s3: add bulk delete api

this is required to support `docker secrets`

.env

@@ -46,6 +46,11 @@ TWILIO_SERVICE_SID=
 # Example: https://app.notesnook.com,http://localhost:3000
 NOTESNOOK_CORS_ORIGINS=
 
+# Description: Add known proxies for incoming HTTP requests
+# Required: no
+# Example: 192.168.1.2,192.168.1.3
+KNOWN_PROXIES=
+
 # Description: This is the public URL for the web app, and is used by the backend for creating redirect URLs (e.g. after email confirmation etc). 
 # Note: the URL has no slashes at the end
 # Required: yes

.github/workflows/publish.yml

@@ -23,12 +23,19 @@ jobs:
         repos:
           - image: streetwriters/notesnook-sync
             file: ./Notesnook.API/Dockerfile
+            context: .
+
+          - image: streetwriters/cors-proxy
+            file: ./cors-proxy/Dockerfile
+            context: ./cors-proxy/
 
           - image: streetwriters/identity
             file: ./Streetwriters.Identity/Dockerfile
+            context: .
 
           - image: streetwriters/sse
             file: ./Streetwriters.Messenger/Dockerfile
+            context: .
     permissions:
       packages: write
       contents: read
@@ -42,7 +49,7 @@ jobs:
       - name: Docker Setup Buildx
         uses: docker/setup-buildx-action@v3
         with:
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64
 
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
@@ -71,10 +78,10 @@ jobs:
         id: push
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: ${{ matrix.repos.context }}
           file: ${{ matrix.repos.file }}
           push: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           cache-from: ${{ matrix.repos.image }}:latest
 

.vscode/launch.json

@@ -9,7 +9,7 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-notesnook",
-      "program": "bin/Debug/net8.0/Notesnook.API.dll",
+      "program": "bin/Debug/net9.0/Notesnook.API.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Notesnook.API",
       "stopAtEntry": false,
@@ -24,7 +24,7 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-identity",
-      "program": "bin/Debug/net8.0/Streetwriters.Identity.dll",
+      "program": "bin/Debug/net9.0/Streetwriters.Identity.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Identity",
       "stopAtEntry": false,
@@ -39,7 +39,7 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-messenger",
-      "program": "bin/Debug/net8.0/Streetwriters.Messenger.dll",
+      "program": "bin/Debug/net9.0/Streetwriters.Messenger.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Messenger",
       "stopAtEntry": false,

Notesnook.API/Accessors/SyncItemsRepositoryAccessor.cs

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
@@ -45,6 +46,8 @@ namespace Notesnook.API.Accessors
         public Repository<Monograph> Monographs { get; }
         public Repository<InboxApiKey> InboxApiKey { get; }
         public Repository<InboxSyncItem> InboxItems { get; }
+        public Repository<SyncDevice> SyncDevices { get; }
+        public Repository<DeviceIdsChunk> DeviceIdsChunks { get; }
 
         public SyncItemsRepositoryAccessor(IDbContext dbContext,
 
@@ -73,25 +76,32 @@ namespace Notesnook.API.Accessors
         [FromKeyedServices(Collections.TagsKey)]
         IMongoCollection<SyncItem> tags,
 
-        Repository<UserSettings> usersSettings, Repository<Monograph> monographs,
-        Repository<InboxApiKey> inboxApiKey, Repository<InboxSyncItem> inboxItems)
+        Repository<UserSettings> usersSettings,
+        Repository<Monograph> monographs,
+        Repository<InboxApiKey> inboxApiKey,
+        Repository<InboxSyncItem> inboxItems,
+        Repository<SyncDevice> syncDevices,
+        Repository<DeviceIdsChunk> deviceIdsChunks,
+        ILogger<SyncItemsRepository> logger)
         {
             UsersSettings = usersSettings;
             Monographs = monographs;
             InboxApiKey = inboxApiKey;
             InboxItems = inboxItems;
-            Notebooks = new SyncItemsRepository(dbContext, notebooks);
-            Notes = new SyncItemsRepository(dbContext, notes);
-            Contents = new SyncItemsRepository(dbContext, content);
-            Settings = new SyncItemsRepository(dbContext, settings);
-            LegacySettings = new SyncItemsRepository(dbContext, legacySettings);
-            Attachments = new SyncItemsRepository(dbContext, attachments);
-            Shortcuts = new SyncItemsRepository(dbContext, shortcuts);
-            Reminders = new SyncItemsRepository(dbContext, reminders);
-            Relations = new SyncItemsRepository(dbContext, relations);
-            Colors = new SyncItemsRepository(dbContext, colors);
-            Vaults = new SyncItemsRepository(dbContext, vaults);
-            Tags = new SyncItemsRepository(dbContext, tags);
+            SyncDevices = syncDevices;
+            DeviceIdsChunks = deviceIdsChunks;
+            Notebooks = new SyncItemsRepository(dbContext, notebooks, logger);
+            Notes = new SyncItemsRepository(dbContext, notes, logger);
+            Contents = new SyncItemsRepository(dbContext, content, logger);
+            Settings = new SyncItemsRepository(dbContext, settings, logger);
+            LegacySettings = new SyncItemsRepository(dbContext, legacySettings, logger);
+            Attachments = new SyncItemsRepository(dbContext, attachments, logger);
+            Shortcuts = new SyncItemsRepository(dbContext, shortcuts, logger);
+            Reminders = new SyncItemsRepository(dbContext, reminders, logger);
+            Relations = new SyncItemsRepository(dbContext, relations, logger);
+            Colors = new SyncItemsRepository(dbContext, colors, logger);
+            Vaults = new SyncItemsRepository(dbContext, vaults, logger);
+            Tags = new SyncItemsRepository(dbContext, tags, logger);
         }
     }
 }

Notesnook.API/Authorization/SyncRequirement.cs

@@ -43,7 +43,7 @@ namespace Notesnook.API.Authorization
             PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
             var result = this.IsAuthorized(context.User, path);
             if (result.Succeeded) context.Succeed(requirement);
-            else if (result.AuthorizationFailure.FailureReasons.Any())
+            else if (result.AuthorizationFailure?.FailureReasons.Any() == true)
                 context.Fail(result.AuthorizationFailure.FailureReasons.First());
             else context.Fail();
 
@@ -63,11 +63,11 @@ namespace Notesnook.API.Authorization
                 return PolicyAuthorizationResult.Forbid(AuthorizationFailure.Failed(reason));
             }
 
-            var hasSyncScope = User.HasClaim("scope", "notesnook.sync");
-            var isInAudience = User.HasClaim("aud", "notesnook");
-            var hasRole = User.HasClaim("role", "notesnook");
+            var hasSyncScope = User?.HasClaim("scope", "notesnook.sync") ?? false;
+            var isInAudience = User?.HasClaim("aud", "notesnook") ?? false;
+            var hasRole = User?.HasClaim("role", "notesnook") ?? false;
 
-            var isEmailVerified = User.HasClaim("verified", "true");
+            var isEmailVerified = User?.HasClaim("verified", "true") ?? false;
 
             if (!isEmailVerified)
             {

Notesnook.API/Constants.cs

@@ -14,7 +14,9 @@ namespace Notesnook.API
         public const string TagsKey = "tags";
         public const string ColorsKey = "colors";
         public const string VaultsKey = "vaults";
-        public const string InboxItems = "inbox_items";
+        public const string InboxItemsKey = "inbox_items";
         public const string InboxApiKeysKey = "inbox_api_keys";
+        public const string SyncDevicesKey = "sync_devices";
+        public const string DeviceIdsChunksKey = "device_ids_chunks";
     }
 }

Notesnook.API/Controllers/AnnouncementController.cs

@@ -21,10 +21,16 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using AngleSharp.Text;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using MongoDB.Driver;
+using Notesnook.API.Accessors;
 using Notesnook.API.Models;
+using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
+using Streetwriters.Common.Interfaces;
+using Streetwriters.Common.Models;
 using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Controllers
@@ -32,38 +38,45 @@ namespace Notesnook.API.Controllers
     // TODO: this should be moved out into its own microservice
     [ApiController]
     [Route("announcements")]
-    public class AnnouncementController : ControllerBase
+    public class AnnouncementController(Repository<Announcement> announcements, WampServiceAccessor serviceAccessor) : ControllerBase
     {
-        private Repository<Announcement> Announcements { get; set; }
-        public AnnouncementController(Repository<Announcement> announcements)
-        {
-            Announcements = announcements;
-        }
-
         [HttpGet("active")]
         [AllowAnonymous]
         public async Task<IActionResult> GetActiveAnnouncements([FromQuery] string? userId)
         {
-            var totalActive = await Announcements.Collection.CountDocumentsAsync(Builders<Announcement>.Filter.Eq("IsActive", true));
-            if (totalActive <= 0) return Ok(Array.Empty<Announcement>());
-
-            var announcements = (await Announcements.FindAsync((a) => a.IsActive)).Where((a) => a.UserIds == null || a.UserIds.Length == 0 || a.UserIds.Contains(userId));
-            foreach (var announcement in announcements)
+            var filter = Builders<Announcement>.Filter.Eq(x => x.IsActive, true);
+            if (!string.IsNullOrEmpty(userId))
             {
-                if (announcement.UserIds != null && !announcement.UserIds.Contains(userId)) continue;
+                var userFilter = Builders<Announcement>.Filter.Or(
+                    Builders<Announcement>.Filter.Eq(x => x.UserIds, null),
+                    Builders<Announcement>.Filter.Size(x => x.UserIds, 0),
+                    Builders<Announcement>.Filter.AnyEq(x => x.UserIds, userId)
+                );
+                filter = Builders<Announcement>.Filter.And(filter, userFilter);
+            }
+            var userAnnouncements = await announcements.Collection.Find(filter).ToListAsync();
+            foreach (var announcement in userAnnouncements)
+            {
+                if (userId != null && announcement.UserIds != null && !announcement.UserIds.Contains(userId)) continue;
 
                 foreach (var item in announcement.Body)
                 {
                     if (item.Type != "callToActions") continue;
                     foreach (var action in item.Actions)
                     {
-                        if (action.Type != "link") continue;
+                        if (action.Type != "link" || action.Data == null) continue;
 
-                        action.Data = action.Data.Replace("{{UserId}}", userId ?? "0");
+                        action.Data = action.Data.Replace("{{UserId}}", userId ?? "");
+
+                        if (action.Data.Contains("{{Email}}"))
+                        {
+                            var user = string.IsNullOrEmpty(userId) ? null : await serviceAccessor.UserAccountService.GetUserAsync(Clients.Notesnook.Id, userId);
+                            action.Data = action.Data.Replace("{{Email}}", user?.Email ?? "");
+                        }
                     }
                 }
             }
-            return Ok(announcements);
+            return Ok(userAnnouncements);
         }
     }
 }

Notesnook.API/Controllers/InboxController.cs

@@ -23,6 +23,7 @@ using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using MongoDB.Bson;
 using Notesnook.API.Authorization;
 using Notesnook.API.Models;
@@ -35,35 +36,27 @@ namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Route("inbox")]
-    public class InboxController : ControllerBase
-    {
-        private readonly Repository<InboxApiKey> InboxApiKey;
-        private readonly Repository<UserSettings> UserSetting;
-        private Repository<InboxSyncItem> InboxItems;
-
-        public InboxController(
+    public class InboxController(
             Repository<InboxApiKey> inboxApiKeysRepository,
             Repository<UserSettings> userSettingsRepository,
-            Repository<InboxSyncItem> inboxItemsRepository)
-        {
-            InboxApiKey = inboxApiKeysRepository;
-            UserSetting = userSettingsRepository;
-            InboxItems = inboxItemsRepository;
-        }
+            Repository<InboxSyncItem> inboxItemsRepository,
+            SyncDeviceService syncDeviceService,
+            ILogger<InboxController> logger) : ControllerBase
+    {
 
         [HttpGet("api-keys")]
         [Authorize(Policy = "Notesnook")]
         public async Task<IActionResult> GetApiKeysAsync()
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
-                var apiKeys = await InboxApiKey.FindAsync(t => t.UserId == userId);
+                var apiKeys = await inboxApiKeysRepository.FindAsync(t => t.UserId == userId);
                 return Ok(apiKeys);
             }
             catch (Exception ex)
             {
-                await Slogger<InboxController>.Error(nameof(GetApiKeysAsync), "Couldn't get inbox api keys.", userId, ex.ToString());
+                logger.LogError(ex, "Couldn't get inbox api keys for user {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -72,7 +65,7 @@ namespace Notesnook.API.Controllers
         [Authorize(Policy = "Notesnook")]
         public async Task<IActionResult> CreateApiKeyAsync([FromBody] InboxApiKey request)
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
                 if (string.IsNullOrWhiteSpace(request.Name))
@@ -84,7 +77,7 @@ namespace Notesnook.API.Controllers
                     return BadRequest(new { error = "Valid expiry date is required." });
                 }
 
-                var count = await InboxApiKey.CountAsync(t => t.UserId == userId);
+                var count = await inboxApiKeysRepository.CountAsync(t => t.UserId == userId);
                 if (count >= 10)
                 {
                     return BadRequest(new { error = "Maximum of 10 inbox api keys allowed." });
@@ -98,12 +91,12 @@ namespace Notesnook.API.Controllers
                     ExpiryDate = request.ExpiryDate,
                     LastUsedAt = 0
                 };
-                await InboxApiKey.InsertAsync(inboxApiKey);
+                await inboxApiKeysRepository.InsertAsync(inboxApiKey);
                 return Ok(inboxApiKey);
             }
             catch (Exception ex)
             {
-                await Slogger<InboxController>.Error(nameof(CreateApiKeyAsync), "Couldn't create inbox api key.", userId, ex.ToString());
+                logger.LogError(ex, "Couldn't create inbox api key for {UserId}.", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -112,7 +105,7 @@ namespace Notesnook.API.Controllers
         [Authorize(Policy = "Notesnook")]
         public async Task<IActionResult> DeleteApiKeyAsync(string apiKey)
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
                 if (string.IsNullOrWhiteSpace(apiKey))
@@ -120,12 +113,12 @@ namespace Notesnook.API.Controllers
                     return BadRequest(new { error = "Api key is required." });
                 }
 
-                await InboxApiKey.DeleteAsync(t => t.UserId == userId && t.Key == apiKey);
+                await inboxApiKeysRepository.DeleteAsync(t => t.UserId == userId && t.Key == apiKey);
                 return Ok(new { message = "Api key deleted successfully." });
             }
             catch (Exception ex)
             {
-                await Slogger<InboxController>.Error(nameof(DeleteApiKeyAsync), "Couldn't delete inbox api key.", userId, ex.ToString());
+                logger.LogError(ex, "Couldn't delete inbox api key for user {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -134,10 +127,10 @@ namespace Notesnook.API.Controllers
         [Authorize(Policy = InboxApiKeyAuthenticationDefaults.AuthenticationScheme)]
         public async Task<IActionResult> GetPublicKeyAsync()
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
-                var userSetting = await UserSetting.FindOneAsync(u => u.UserId == userId);
+                var userSetting = await userSettingsRepository.FindOneAsync(u => u.UserId == userId);
                 if (string.IsNullOrWhiteSpace(userSetting?.InboxKeys?.Public))
                 {
                     return BadRequest(new { error = "Inbox public key is not configured." });
@@ -146,7 +139,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<InboxController>.Error(nameof(GetPublicKeyAsync), "Couldn't get user's inbox's public key.", userId, ex.ToString());
+                logger.LogError(ex, "Couldn't get user's inbox's public key for user {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -155,7 +148,7 @@ namespace Notesnook.API.Controllers
         [Authorize(Policy = InboxApiKeyAuthenticationDefaults.AuthenticationScheme)]
         public async Task<IActionResult> CreateInboxItemAsync([FromBody] InboxSyncItem request)
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
                 if (request.Key.Algorithm != Algorithms.XSAL_X25519_7)
@@ -189,9 +182,8 @@ namespace Notesnook.API.Controllers
 
                 request.UserId = userId;
                 request.ItemId = ObjectId.GenerateNewId().ToString();
-                await InboxItems.InsertAsync(request);
-                new SyncDeviceService(new SyncDevice(userId, string.Empty))
-                    .AddIdsToAllDevices([$"{request.ItemId}:inboxItems"]);
+                await inboxItemsRepository.InsertAsync(request);
+                await syncDeviceService.AddIdsToAllDevicesAsync(userId, [new(request.ItemId, "inbox_item")]);
                 await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
                 {
                     OriginTokenId = null,
@@ -206,7 +198,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<InboxController>.Error(nameof(CreateInboxItemAsync), "Couldn't create inbox item.", userId, ex.ToString());
+                logger.LogError(ex, "Couldn't create inbox item for user {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }

Notesnook.API/Controllers/MonographsController.cs

@@ -26,13 +26,16 @@ using System.Threading.Tasks;
 using AngleSharp;
 using AngleSharp.Dom;
 using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using Notesnook.API.Authorization;
 using Notesnook.API.Models;
 using Notesnook.API.Services;
 using Streetwriters.Common;
+using Streetwriters.Common.Helpers;
 using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Data.Interfaces;
@@ -43,7 +46,7 @@ namespace Notesnook.API.Controllers
     [ApiController]
     [Route("monographs")]
     [Authorize("Sync")]
-    public class MonographsController(Repository<Monograph> monographs, IURLAnalyzer analyzer) : ControllerBase
+    public class MonographsController(Repository<Monograph> monographs, IURLAnalyzer analyzer, SyncDeviceService syncDeviceService, ILogger<MonographsController> logger) : ControllerBase
     {
         const string SVG_PIXEL = "<svg xmlns='http://www.w3.org/2000/svg' width='1' height='1'><circle r='9'/></svg>";
         private const int MAX_DOC_SIZE = 15 * 1024 * 1024;
@@ -97,9 +100,8 @@ namespace Notesnook.API.Controllers
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
+                var userId = this.User.GetUserId();
                 var jti = this.User.FindFirstValue("jti");
-                if (userId == null) return Unauthorized();
 
                 var existingMonograph = await FindMonographAsync(userId, monograph);
                 if (existingMonograph != null && !existingMonograph.Deleted) return await UpdateAsync(deviceId, monograph);
@@ -117,6 +119,7 @@ namespace Notesnook.API.Controllers
                     monograph.Id = existingMonograph.Id;
                 }
                 monograph.Deleted = false;
+                monograph.ViewCount = 0;
                 await monographs.Collection.ReplaceOneAsync(
                     CreateMonographFilter(userId, monograph),
                     monograph,
@@ -128,12 +131,12 @@ namespace Notesnook.API.Controllers
                 return Ok(new
                 {
                     id = monograph.ItemId,
-                    datePublished = monograph.DatePublished,
+                    datePublished = monograph.DatePublished
                 });
             }
             catch (Exception e)
             {
-                await Slogger<MonographsController>.Error(nameof(PublishAsync), e.ToString());
+                logger.LogError(e, "Failed to publish monograph");
                 return BadRequest();
             }
         }
@@ -143,9 +146,8 @@ namespace Notesnook.API.Controllers
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
+                var userId = this.User.GetUserId();
                 var jti = this.User.FindFirstValue("jti");
-                if (userId == null) return Unauthorized();
 
                 var existingMonograph = await FindMonographAsync(userId, monograph);
                 if (existingMonograph == null || existingMonograph.Deleted)
@@ -179,12 +181,12 @@ namespace Notesnook.API.Controllers
                 return Ok(new
                 {
                     id = monograph.ItemId,
-                    datePublished = monograph.DatePublished,
+                    datePublished = monograph.DatePublished
                 });
             }
             catch (Exception e)
             {
-                await Slogger<MonographsController>.Error(nameof(UpdateAsync), e.ToString());
+                logger.LogError(e, "Failed to update monograph");
                 return BadRequest();
             }
         }
@@ -192,8 +194,7 @@ namespace Notesnook.API.Controllers
         [HttpGet]
         public async Task<IActionResult> GetUserMonographsAsync()
         {
-            var userId = this.User.FindFirstValue("sub");
-            if (userId == null) return Unauthorized();
+            var userId = this.User.GetUserId();
 
             var userMonographs = (await monographs.Collection.FindAsync(
                     Builders<Monograph>.Filter.And(
@@ -234,6 +235,9 @@ namespace Notesnook.API.Controllers
             var monograph = await FindMonographAsync(id);
             if (monograph == null || monograph.Deleted) return Content(SVG_PIXEL, "image/svg+xml");
 
+            var cookieName = $"viewed_{id}";
+            var hasVisitedBefore = Request.Cookies.ContainsKey(cookieName);
+
             if (monograph.SelfDestruct)
             {
                 await monographs.Collection.ReplaceOneAsync(
@@ -243,21 +247,52 @@ namespace Notesnook.API.Controllers
                         ItemId = id,
                         Id = monograph.Id,
                         Deleted = true,
-                        UserId = monograph.UserId
+                        UserId = monograph.UserId,
+                        ViewCount = 0
                     }
                 );
-
                 await MarkMonographForSyncAsync(monograph.UserId, id);
             }
+            else if (!hasVisitedBefore)
+            {
+                await monographs.Collection.UpdateOneAsync(
+                    CreateMonographFilter(monograph.UserId, monograph),
+                    Builders<Monograph>.Update.Inc(m => m.ViewCount, 1)
+                );
+
+                var cookieOptions = new CookieOptions
+                {
+                    Path = $"/monographs/{id}",
+                    HttpOnly = true,
+                    Secure = Request.IsHttps,
+                    Expires = DateTimeOffset.UtcNow.AddMonths(1)
+                };
+                Response.Cookies.Append(cookieName, "1", cookieOptions);
+            }
 
             return Content(SVG_PIXEL, "image/svg+xml");
         }
 
+        [HttpGet("{id}/analytics")]
+        public async Task<IActionResult> GetMonographAnalyticsAsync([FromRoute] string id)
+        {
+            if (!FeatureAuthorizationHelper.IsFeatureAllowed(Features.MONOGRAPH_ANALYTICS, Clients.Notesnook.Id, User))
+                return BadRequest(new { error = "Monograph analytics are only available on the Pro & Believer plans." });
+
+            var userId = this.User.GetUserId();
+            var monograph = await FindMonographAsync(id);
+            if (monograph == null || monograph.Deleted || monograph.UserId != userId)
+            {
+                return NotFound();
+            }
+
+            return Ok(new { totalViews = monograph.ViewCount });
+        }
+
         [HttpDelete("{id}")]
         public async Task<IActionResult> DeleteAsync([FromQuery] string? deviceId, [FromRoute] string id)
         {
-            var userId = this.User.FindFirstValue("sub");
-            if (userId is null) return Unauthorized();
+            var userId = this.User.GetUserId();
 
             var monograph = await FindMonographAsync(id);
             if (monograph == null || monograph.Deleted)
@@ -272,7 +307,8 @@ namespace Notesnook.API.Controllers
                     ItemId = id,
                     Id = monograph.Id,
                     Deleted = true,
-                    UserId = monograph.UserId
+                    UserId = monograph.UserId,
+                    ViewCount = 0
                 }
             );
 
@@ -281,40 +317,25 @@ namespace Notesnook.API.Controllers
             return Ok();
         }
 
-        private static async Task MarkMonographForSyncAsync(string userId, string monographId, string? deviceId, string? jti)
+        private async Task MarkMonographForSyncAsync(string userId, string monographId, string? deviceId, string? jti)
         {
             if (deviceId == null) return;
 
-            new SyncDeviceService(new SyncDevice(userId, deviceId)).AddIdsToOtherDevices([$"{monographId}:monograph"]);
-            await SendTriggerSyncEventAsync(userId, jti);
+            await syncDeviceService.AddIdsToOtherDevicesAsync(userId, deviceId, [new(monographId, "monograph")]);
         }
 
-        private static async Task MarkMonographForSyncAsync(string userId, string monographId)
+        private async Task MarkMonographForSyncAsync(string userId, string monographId)
         {
-            new SyncDeviceService(new SyncDevice(userId, string.Empty)).AddIdsToAllDevices([$"{monographId}:monograph"]);
-            await SendTriggerSyncEventAsync(userId, sendToAllDevices: true);
+            await syncDeviceService.AddIdsToAllDevicesAsync(userId, [new(monographId, "monograph")]);
         }
 
-        private static async Task SendTriggerSyncEventAsync(string userId, string? jti = null, bool sendToAllDevices = false)
-        {
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
-            {
-                OriginTokenId = sendToAllDevices ? null : jti,
-                UserId = userId,
-                Message = new Message
-                {
-                    Type = "triggerSync",
-                    Data = JsonSerializer.Serialize(new { reason = "Monographs updated." })
-                }
-            });
-        }
-
-        private async Task<string> CleanupContentAsync(ClaimsPrincipal user, string content)
+        private async Task<string> CleanupContentAsync(ClaimsPrincipal user, string? content)
         {
+            if (string.IsNullOrEmpty(content)) return string.Empty;
             if (Constants.IS_SELF_HOSTED) return content;
             try
             {
-                var json = JsonSerializer.Deserialize<MonographContent>(content);
+                var json = JsonSerializer.Deserialize<MonographContent>(content) ?? throw new Exception("Invalid monograph content.");
                 var html = json.Data;
 
                 if (user.IsUserSubscribed())
@@ -328,7 +349,7 @@ namespace Notesnook.API.Controllers
                         if (string.IsNullOrEmpty(href)) continue;
                         if (!await analyzer.IsURLSafeAsync(href))
                         {
-                            await Slogger<MonographsController>.Info("CleanupContentAsync", "Malicious URL detected: " + href);
+                            logger.LogInformation("Malicious URL detected: {Url}", href);
                             element.RemoveAttribute("href");
                         }
                     }
@@ -355,7 +376,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<MonographsController>.Error("CleanupContentAsync", ex.ToString());
+                logger.LogError(ex, "Failed to cleanup monograph content");
                 return content;
             }
         }

Notesnook.API/Controllers/S3Controller.cs

@@ -17,20 +17,22 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Mvc;
-using Amazon.S3.Model;
-using System.Threading.Tasks;
-using System.Security.Claims;
-using Notesnook.API.Interfaces;
 using System;
 using System.Net.Http;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http.Extensions;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Notesnook.API.Helpers;
+using Notesnook.API.Interfaces;
+using Notesnook.API.Models;
+using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
 using Streetwriters.Common.Extensions;
 using Streetwriters.Common.Models;
-using Notesnook.API.Helpers;
-using Streetwriters.Common;
-using Streetwriters.Common.Interfaces;
-using Notesnook.API.Models;
 
 namespace Notesnook.API.Controllers
 {
@@ -38,102 +40,124 @@ namespace Notesnook.API.Controllers
     [Route("s3")]
     [ResponseCache(NoStore = true, Location = ResponseCacheLocation.None)]
     [Authorize("Sync")]
-    public class S3Controller : ControllerBase
+    public class S3Controller(IS3Service s3Service, ISyncItemsRepositoryAccessor repositories, WampServiceAccessor serviceAccessor, ILogger<S3Controller> logger) : ControllerBase
     {
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private IS3Service S3Service { get; set; }
-        public S3Controller(IS3Service s3Service, ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor)
-        {
-            S3Service = s3Service;
-            Repositories = syncItemsRepositoryAccessor;
-        }
-
         [HttpPut]
         public async Task<IActionResult> Upload([FromQuery] string name)
         {
-            var userId = this.User.FindFirstValue("sub");
-
-            if (!HttpContext.Request.Headers.ContentLength.HasValue) return BadRequest(new { error = "No Content-Length header found." });
-
-            long fileSize = HttpContext.Request.Headers.ContentLength.Value;
-            if (fileSize == 0)
+            try
             {
-                var uploadUrl = S3Service.GetUploadObjectUrl(userId, name);
-                if (uploadUrl == null) return BadRequest(new { error = "Could not create signed url." });
-                return Ok(uploadUrl);
-            }
+                var userId = this.User.GetUserId();
 
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            if (!Constants.IS_SELF_HOSTED)
-            {
-                var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
-                var subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId);
-                if (subscription is null) return BadRequest(new { error = "User subscription not found." });
+                var fileSize = HttpContext.Request.ContentLength ?? 0;
+                bool hasBody = fileSize > 0;
 
-                if (StorageHelper.IsFileSizeExceeded(subscription, fileSize))
+                if (!hasBody)
                 {
-                    return BadRequest(new { error = "Max file size exceeded." });
+                    return Ok(Request.GetEncodedUrl() + "&access_token=" + Request.Headers.Authorization.ToString().Replace("Bearer ", ""));
                 }
 
-                userSettings.StorageLimit ??= new Limit { Value = 0, UpdatedAt = 0 };
-                userSettings.StorageLimit.Value += fileSize;
-                if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit))
-                    return BadRequest(new { error = "Storage limit exceeded." });
-            }
+                if (Constants.IS_SELF_HOSTED) await UploadFileAsync(userId, name, fileSize);
+                else await UploadFileWithChecksAsync(userId, name, fileSize);
 
-            var url = S3Service.GetInternalUploadObjectUrl(userId, name);
-            if (url == null) return BadRequest(new { error = "Could not create signed url." });
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error uploading attachment for user.");
+                return BadRequest(new { error = "Failed to upload attachment." });
+            }
+        }
+
+        private async Task UploadFileWithChecksAsync(string userId, string name, long fileSize)
+        {
+            var userSettings = await repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+
+            var subscription = await serviceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
+
+            if (StorageHelper.IsFileSizeExceeded(subscription, fileSize))
+                throw new Exception("Max file size exceeded.");
+
+            userSettings.StorageLimit = StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+            if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value + fileSize))
+                throw new Exception("Storage limit exceeded.");
+
+            var uploadedFileSize = await UploadFileAsync(userId, name, fileSize);
+
+            userSettings.StorageLimit.Value += uploadedFileSize;
+            await repositories.UsersSettings.Collection.UpdateOneAsync(
+                Builders<UserSettings>.Filter.Eq(u => u.UserId, userId),
+                Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
+            );
+
+            // extra check in case user sets wrong ContentLength in the HTTP header
+            if (uploadedFileSize != fileSize && StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value))
+            {
+                await s3Service.DeleteObjectAsync(userId, name);
+                throw new Exception("Storage limit exceeded.");
+            }
+        }
+
+        private async Task<long> UploadFileAsync(string userId, string name, long fileSize)
+        {
+            var url = await s3Service.GetInternalUploadObjectUrlAsync(userId, name) ?? throw new Exception("Could not create signed url.");
 
             var httpClient = new HttpClient();
             var content = new StreamContent(HttpContext.Request.BodyReader.AsStream());
-            content.Headers.ContentLength = Request.ContentLength;
+            content.Headers.ContentLength = fileSize;
             var response = await httpClient.SendRequestAsync<Response>(url, null, HttpMethod.Put, content);
-            if (!response.Success) return BadRequest(await response.Content.ReadAsStringAsync());
+            if (!response.Success) throw new Exception(response.Content != null ? await response.Content.ReadAsStringAsync() : "Could not upload file.");
 
-            if (!Constants.IS_SELF_HOSTED)
-            {
-                userSettings.StorageLimit.UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-            }
-
-            return Ok(response);
+            return await s3Service.GetObjectSizeAsync(userId, name);
         }
 
 
         [HttpGet("multipart")]
         public async Task<IActionResult> MultipartUpload([FromQuery] string name, [FromQuery] int parts, [FromQuery] string? uploadId)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
             try
             {
-                var meta = await S3Service.StartMultipartUploadAsync(userId, name, parts, uploadId);
+                var meta = await s3Service.StartMultipartUploadAsync(userId, name, parts, uploadId);
                 return Ok(meta);
             }
-            catch (Exception ex) { return BadRequest(ex.Message); }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error starting multipart upload for user.");
+                return BadRequest(new { error = "Failed to start multipart upload." });
+            }
         }
 
         [HttpDelete("multipart")]
         public async Task<IActionResult> AbortMultipartUpload([FromQuery] string name, [FromQuery] string uploadId)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
             try
             {
-                await S3Service.AbortMultipartUploadAsync(userId, name, uploadId);
+                await s3Service.AbortMultipartUploadAsync(userId, name, uploadId);
                 return Ok();
             }
-            catch (Exception ex) { return BadRequest(ex.Message); }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error aborting multipart upload for user.");
+                return BadRequest(new { error = "Failed to abort multipart upload." });
+            }
         }
 
         [HttpPost("multipart")]
         public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequestWrapper uploadRequestWrapper)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
             try
             {
-                await S3Service.CompleteMultipartUploadAsync(userId, uploadRequestWrapper.ToRequest());
+                await s3Service.CompleteMultipartUploadAsync(userId, uploadRequestWrapper.ToRequest());
                 return Ok();
             }
-            catch (Exception ex) { return BadRequest(ex.Message); }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error completing multipart upload for user.");
+                return BadRequest(new { error = "Failed to complete multipart upload." });
+            }
         }
 
         [HttpGet]
@@ -141,21 +165,33 @@ namespace Notesnook.API.Controllers
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
-                var url = await S3Service.GetDownloadObjectUrl(userId, name);
+                var userId = this.User.GetUserId();
+                var url = await s3Service.GetDownloadObjectUrlAsync(userId, name);
                 if (url == null) return BadRequest("Could not create signed url.");
                 return Ok(url);
             }
-            catch (Exception ex) { return BadRequest(ex.Message); }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error generating download url for user.");
+                return BadRequest(new { error = "Failed to get attachment url." });
+            }
         }
 
         [HttpHead]
         public async Task<IActionResult> Info([FromQuery] string name)
         {
-            var userId = this.User.FindFirstValue("sub");
-            var size = await S3Service.GetObjectSizeAsync(userId, name);
-            HttpContext.Response.Headers.ContentLength = size;
-            return Ok();
+            try
+            {
+                var userId = this.User.GetUserId();
+                var size = await s3Service.GetObjectSizeAsync(userId, name);
+                HttpContext.Response.Headers.ContentLength = size;
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error getting object info for user.");
+                return BadRequest(new { error = "Failed to get attachment info." });
+            }
         }
 
         [HttpDelete]
@@ -163,13 +199,35 @@ namespace Notesnook.API.Controllers
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
-                await S3Service.DeleteObjectAsync(userId, name);
+                var userId = this.User.GetUserId();
+                await s3Service.DeleteObjectAsync(userId, name);
                 return Ok();
             }
             catch (Exception ex)
             {
-                return BadRequest(ex.Message);
+                logger.LogError(ex, "Error deleting object for user.");
+                return BadRequest(new { error = "Failed to delete attachment." });
+            }
+        }
+
+        [HttpPost("bulk-delete")]
+        public async Task<IActionResult> DeleteBulkAsync([FromBody] DeleteBulkObjectsRequest request)
+        {
+            try
+            {
+                if (request.Names == null || request.Names.Length == 0)
+                {
+                    return BadRequest(new { error = "No files specified for deletion." });
+                }
+
+                var userId = this.User.GetUserId();
+                await s3Service.DeleteObjectsAsync(userId, request.Names);
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error deleting objects for user.");
+                return BadRequest(new { error = "Failed to delete attachments." });
             }
         }
     }

Notesnook.API/Controllers/SyncDeviceController.cs

@@ -24,6 +24,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models.Responses;
 using Notesnook.API.Services;
@@ -36,20 +37,20 @@ namespace Notesnook.API.Controllers
     [ApiController]
     [Authorize]
     [Route("devices")]
-    public class SyncDeviceController : ControllerBase
+    public class SyncDeviceController(SyncDeviceService syncDeviceService, ILogger<SyncDeviceController> logger) : ControllerBase
     {
         [HttpPost]
         public async Task<IActionResult> RegisterDevice([FromQuery] string deviceId)
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub") ?? throw new Exception("User not found.");
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).RegisterDevice();
+                var userId = this.User.GetUserId();
+                await syncDeviceService.RegisterDeviceAsync(userId, deviceId);
                 return Ok();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't register device.", ex.ToString());
+                logger.LogError(ex, "Failed to register device: {DeviceId}", deviceId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -60,13 +61,13 @@ namespace Notesnook.API.Controllers
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub") ?? throw new Exception("User not found.");
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).UnregisterDevice();
+                var userId = this.User.GetUserId();
+                await syncDeviceService.UnregisterDeviceAsync(userId, deviceId);
                 return Ok();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't unregister device.", ex.ToString());
+                logger.LogError(ex, "Failed to unregister device: {DeviceId}", deviceId);
                 return BadRequest(new { error = ex.Message });
             }
         }

Notesnook.API/Controllers/UsersController.cs

@@ -23,6 +23,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http.Timeouts;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
@@ -33,7 +34,7 @@ namespace Notesnook.API.Controllers
     [ApiController]
     [Authorize]
     [Route("users")]
-    public class UsersController(IUserService UserService) : ControllerBase
+    public class UsersController(IUserService UserService, ILogger<UsersController> logger) : ControllerBase
     {
         [HttpPost]
         [AllowAnonymous]
@@ -46,7 +47,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(Signup), "Couldn't sign up.", ex.ToString());
+                logger.LogError(ex, "Failed to sign up user");
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -54,7 +55,7 @@ namespace Notesnook.API.Controllers
         [HttpGet]
         public async Task<IActionResult> GetUser()
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
                 UserResponse response = await UserService.GetUserAsync(userId);
@@ -63,7 +64,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't get user for id.", userId, ex.ToString());
+                logger.LogError(ex, "Failed to get user with id: {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -71,7 +72,7 @@ namespace Notesnook.API.Controllers
         [HttpPatch]
         public async Task<IActionResult> UpdateUser([FromBody] UserKeys keys)
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
                 await UserService.SetUserKeysAsync(userId, keys);
@@ -79,7 +80,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't update user with id.", userId, ex.ToString());
+                logger.LogError(ex, "Failed to update user with id: {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -87,7 +88,7 @@ namespace Notesnook.API.Controllers
         [HttpPost("reset")]
         public async Task<IActionResult> Reset([FromForm] bool removeAttachments)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
 
             if (await UserService.ResetUserAsync(userId, removeAttachments))
                 return Ok();
@@ -98,7 +99,7 @@ namespace Notesnook.API.Controllers
         [RequestTimeout(5 * 60 * 1000)]
         public async Task<IActionResult> Delete([FromForm] DeleteAccountForm form)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
             var jti = User.FindFirstValue("jti");
             try
             {
@@ -107,7 +108,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't delete user with id.", userId, ex.ToString());
+                logger.LogError(ex, "Failed to delete user with id: {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }

Notesnook.API/Dockerfile

@@ -1,7 +1,7 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/runtime-deps:9.0-alpine AS base
 WORKDIR /app
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-alpine AS build
 ARG TARGETARCH
 ARG BUILDPLATFORM
 ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
@@ -27,8 +27,6 @@ FROM build AS publish
 RUN dotnet publish -c Release -o /app/publish \
   #--runtime alpine-x64 \
   --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
   /p:PublishSingleFile=true \
   /p:JsonSerializerIsReflectionEnabledByDefault=true \
   -a $TARGETARCH

Notesnook.API/EventSources/SyncEventCounterSource.cs

@@ -7,19 +7,17 @@ public sealed class SyncEventCounterSource : EventSource
 {
     public static readonly SyncEventCounterSource Log = new();
 
-    private Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
-    private Counter<int> fetchCounter;
-    private Counter<int> pushCounter;
-    private Counter<int> legacyFetchCounter;
-    private Counter<int> pushV2Counter;
-    private Counter<int> fetchV2Counter;
-    private Histogram<long> fetchV2Duration;
-    private Histogram<long> pushV2Duration;
+    private readonly Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
+    private readonly Counter<int> fetchCounter;
+    private readonly Counter<int> pushCounter;
+    private readonly Counter<int> pushV2Counter;
+    private readonly Counter<int> fetchV2Counter;
+    private readonly Histogram<long> fetchV2Duration;
+    private readonly Histogram<long> pushV2Duration;
     private SyncEventCounterSource()
     {
         fetchCounter = meter.CreateCounter<int>("sync.fetches", "fetches", "Total fetches");
         pushCounter = meter.CreateCounter<int>("sync.pushes", "pushes", "Total pushes");
-        legacyFetchCounter = meter.CreateCounter<int>("sync.legacy-fetches", "fetches", "Total legacy fetches");
         fetchV2Counter = meter.CreateCounter<int>("sync.v2.fetches", "fetches", "Total v2 fetches");
         pushV2Counter = meter.CreateCounter<int>("sync.v2.pushes", "pushes", "Total v2 pushes");
         fetchV2Duration = meter.CreateHistogram<long>("sync.v2.fetch_duration");
@@ -27,7 +25,6 @@ public sealed class SyncEventCounterSource : EventSource
     }
 
     public void Fetch() => fetchCounter.Add(1);
-    public void LegacyFetch() => legacyFetchCounter.Add(1);
     public void FetchV2() => fetchV2Counter.Add(1);
     public void PushV2() => pushV2Counter.Add(1);
     public void Push() => pushCounter.Add(1);
@@ -36,14 +33,7 @@ public sealed class SyncEventCounterSource : EventSource
 
     protected override void Dispose(bool disposing)
     {
-        legacyFetchCounter = null;
-        fetchV2Counter = null;
-        pushV2Counter = null;
-        pushCounter = null;
-        fetchCounter = null;
         meter.Dispose();
-        meter = null;
-
         base.Dispose(disposing);
     }
 }

Notesnook.API/Extensions/ClaimsPrincipalExtensions.cs

@@ -10,5 +10,8 @@ namespace System.Security.Claims
         private readonly static string[] SUBSCRIBED_CLAIMS = ["believer", "education", "essential", "pro", "legacy_pro"];
         public static bool IsUserSubscribed(this ClaimsPrincipal user)
          => user.Claims.Any((c) => c.Type == "notesnook:status" && SUBSCRIBED_CLAIMS.Contains(c.Value));
+
+        public static string GetUserId(this ClaimsPrincipal user)
+        => user.FindFirstValue("sub") ?? throw new Exception("User ID not found in claims.");
     }
 }

Notesnook.API/Helpers/S3FailoverHelper.cs

@@ -0,0 +1,414 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Microsoft.Extensions.Logging;
+
+namespace Notesnook.API.Helpers
+{
+    /// <summary>
+    /// Configuration for S3 failover behavior
+    /// </summary>
+    public class S3FailoverConfig
+    {
+        /// <summary>
+        /// Maximum number of retry attempts per endpoint
+        /// </summary>
+        public int MaxRetries { get; set; } = 3;
+
+        /// <summary>
+        /// Delay between retries in milliseconds
+        /// </summary>
+        public int RetryDelayMs { get; set; } = 1000;
+
+        /// <summary>
+        /// Whether to use exponential backoff for retries
+        /// </summary>
+        public bool UseExponentialBackoff { get; set; } = true;
+
+        /// <summary>
+        /// Whether to allow failover for write operations (PUT, POST, DELETE).
+        /// Default is false to prevent data consistency issues.
+        /// </summary>
+        public bool AllowWriteFailover { get; set; } = false;
+
+        /// <summary>
+        /// List of exception types that should trigger failover
+        /// </summary>
+        public HashSet<Type> FailoverExceptions { get; set; } = new()
+        {
+            typeof(AmazonS3Exception),
+            typeof(System.Net.Http.HttpRequestException),
+            typeof(System.Net.Sockets.SocketException),
+            typeof(System.Threading.Tasks.TaskCanceledException),
+            typeof(TimeoutException)
+        };
+
+        /// <summary>
+        /// List of S3 error codes that should trigger failover
+        /// </summary>
+        public HashSet<string> FailoverErrorCodes { get; set; } = new()
+        {
+            "ServiceUnavailable",
+            "SlowDown",
+            "InternalError",
+            "RequestTimeout"
+        };
+    }
+
+    /// <summary>
+    /// Result of a failover operation
+    /// </summary>
+    public class S3FailoverResult<T>
+    {
+        public T? Result { get; set; }
+        public bool UsedFailover { get; set; }
+        public int ClientIndex { get; set; } = 0;
+        public int AttemptsUsed { get; set; }
+        public Exception? LastException { get; set; }
+    }
+
+    /// <summary>
+    /// Helper class for S3 operations with automatic failover to multiple endpoints
+    /// </summary>
+    public class S3FailoverHelper
+    {
+        private readonly List<AmazonS3Client> clients;
+        private readonly S3FailoverConfig config;
+        private readonly ILogger? logger;
+
+        /// <summary>
+        /// Initialize with a list of S3 clients (first is primary, rest are failover endpoints)
+        /// </summary>
+        public S3FailoverHelper(
+            IEnumerable<AmazonS3Client> clients,
+            S3FailoverConfig? config = null,
+            ILogger? logger = null)
+        {
+            if (clients == null) throw new ArgumentNullException(nameof(clients));
+            this.clients = new List<AmazonS3Client>(clients);
+            if (this.clients.Count == 0) throw new ArgumentException("At least one S3 client is required", nameof(clients));
+            this.config = config ?? new S3FailoverConfig();
+            this.logger = logger;
+        }
+
+        /// <summary>
+        /// Initialize with params array of S3 clients
+        /// </summary>
+        public S3FailoverHelper(
+            S3FailoverConfig? config = null,
+            ILogger? logger = null,
+            params AmazonS3Client[] clients)
+        {
+            if (clients == null || clients.Length == 0)
+                throw new ArgumentException("At least one S3 client is required", nameof(clients));
+            this.clients = new List<AmazonS3Client>(clients);
+            this.config = config ?? new S3FailoverConfig();
+            this.logger = logger;
+        }
+
+        /// <summary>
+        /// Execute an S3 operation with automatic failover
+        /// </summary>
+        /// <param name="operation">The S3 operation to execute</param>
+        /// <param name="operationName">Name of the operation for logging</param>
+        /// <param name="isWriteOperation">Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default.</param>
+        public async Task<T> ExecuteWithFailoverAsync<T>(
+            Func<AmazonS3Client, Task<T>> operation,
+            string operationName = "S3Operation",
+            bool isWriteOperation = false)
+        {
+            var result = await ExecuteWithFailoverInternalAsync(operation, operationName, isWriteOperation);
+            if (result.Result == null)
+            {
+                throw result.LastException ?? new Exception($"Failed to execute {operationName} on all endpoints");
+            }
+            return result.Result;
+        }
+
+        /// <summary>
+        /// Execute an S3 operation with automatic failover and return detailed result
+        /// </summary>
+        /// <param name="operation">The S3 operation to execute</param>
+        /// <param name="operationName">Name of the operation for logging</param>
+        /// <param name="isWriteOperation">Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default.</param>
+        private async Task<S3FailoverResult<T>> ExecuteWithFailoverInternalAsync<T>(
+            Func<AmazonS3Client, Task<T>> operation,
+            string operationName = "S3Operation",
+            bool isWriteOperation = false)
+        {
+            var result = new S3FailoverResult<T>();
+            Exception? lastException = null;
+
+            // Determine max clients to try based on write operation flag
+            var maxClientsToTry = (isWriteOperation && !config.AllowWriteFailover) ? 1 : clients.Count;
+
+            if (isWriteOperation && !config.AllowWriteFailover && clients.Count > 1)
+            {
+                logger?.LogDebug(
+                    "Write operation {Operation} will only use primary endpoint. Failover is disabled for write operations.",
+                    operationName);
+            }
+
+            // Try each client in sequence (first is primary, rest are failovers)
+            for (int i = 0; i < maxClientsToTry; i++)
+            {
+                var client = clients[i];
+                var clientName = i == 0 ? "primary" : $"failover-{i}";
+                var isPrimary = i == 0;
+
+                if (!isPrimary && lastException != null)
+                {
+                    logger?.LogWarning(lastException,
+                        "Previous S3 endpoint failed for {Operation}. Attempting {ClientName} (endpoint {Index}/{Total}).",
+                        operationName, clientName, i + 1, maxClientsToTry);
+                }
+
+                var (success, value, exception, attempts) = await TryExecuteAsync(client, operation, operationName, clientName);
+                result.AttemptsUsed += attempts;
+
+                if (success && value != null)
+                {
+                    result.Result = value;
+                    result.UsedFailover = !isPrimary;
+                    result.ClientIndex = i;
+
+                    if (!isPrimary)
+                    {
+                        logger?.LogInformation(
+                            "Successfully failed over to {ClientName} S3 endpoint for {Operation}",
+                            clientName, operationName);
+                    }
+
+                    return result;
+                }
+
+                lastException = exception;
+
+                // If this is not the last client and should retry, log and continue
+                if (i < maxClientsToTry - 1 && ShouldFailover(exception))
+                {
+                    logger?.LogWarning(exception,
+                        "Endpoint {ClientName} failed for {Operation}. {Remaining} endpoint(s) remaining.",
+                        clientName, operationName, maxClientsToTry - i - 1);
+                }
+            }
+
+            // All clients failed
+            result.LastException = lastException;
+            logger?.LogError(lastException,
+                "All S3 endpoints failed for {Operation}. Total endpoints tried: {EndpointCount}, Total attempts: {Attempts}",
+                operationName, maxClientsToTry, result.AttemptsUsed);
+
+            return result;
+        }        /// <summary>
+                 /// Try to execute an operation with retries
+                 /// </summary>
+        private async Task<(bool success, T? value, Exception? exception, int attempts)> TryExecuteAsync<T>(
+            AmazonS3Client client,
+            Func<AmazonS3Client, Task<T>> operation,
+            string operationName,
+            string endpointName)
+        {
+            Exception? lastException = null;
+            int attempts = 0;
+
+            for (int retry = 0; retry <= config.MaxRetries; retry++)
+            {
+                attempts++;
+                try
+                {
+                    var result = await operation(client);
+                    return (true, result, null, attempts);
+                }
+                catch (Exception ex)
+                {
+                    lastException = ex;
+
+                    if (retry < config.MaxRetries && ShouldRetry(ex))
+                    {
+                        var delay = CalculateRetryDelay(retry);
+                        logger?.LogWarning(ex,
+                            "Attempt {Attempt}/{MaxAttempts} failed for {Operation} on {Endpoint}. Retrying in {Delay}ms",
+                            retry + 1, config.MaxRetries + 1, operationName, endpointName, delay);
+
+                        await Task.Delay(delay);
+                    }
+                    else
+                    {
+                        logger?.LogError(ex,
+                            "Operation {Operation} failed on {Endpoint} after {Attempts} attempts",
+                            operationName, endpointName, attempts);
+                        break;
+                    }
+                }
+            }
+
+            return (false, default, lastException, attempts);
+        }
+
+        /// <summary>
+        /// Determine if an exception should trigger a retry
+        /// </summary>
+        private bool ShouldRetry(Exception exception)
+        {
+            // Check if exception type is in the retry list
+            var exceptionType = exception.GetType();
+            if (config.FailoverExceptions.Contains(exceptionType))
+            {
+                // For S3 exceptions, check error codes
+                if (exception is AmazonS3Exception s3Exception)
+                {
+                    return config.FailoverErrorCodes.Contains(s3Exception.ErrorCode);
+                }
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Determine if an exception should trigger failover to secondary endpoint
+        /// </summary>
+        private bool ShouldFailover(Exception? exception)
+        {
+            if (exception == null) return false;
+            return ShouldRetry(exception);
+        }
+
+        /// <summary>
+        /// Calculate delay for retry based on retry attempt number
+        /// </summary>
+        private int CalculateRetryDelay(int retryAttempt)
+        {
+            if (!config.UseExponentialBackoff)
+            {
+                return config.RetryDelayMs;
+            }
+
+            // Exponential backoff: delay * 2^retryAttempt
+            return config.RetryDelayMs * (int)Math.Pow(2, retryAttempt);
+        }
+
+        /// <summary>
+        /// Execute a void operation with automatic failover
+        /// </summary>
+        /// <param name="operation">The S3 operation to execute</param>
+        /// <param name="operationName">Name of the operation for logging</param>
+        /// <param name="isWriteOperation">Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default.</param>
+        public async Task ExecuteWithFailoverAsync(
+            Func<AmazonS3Client, Task> operation,
+            string operationName = "S3Operation",
+            bool isWriteOperation = false)
+        {
+            await ExecuteWithFailoverAsync<object?>(async (client) =>
+            {
+                await operation(client);
+                return null;
+            }, operationName, isWriteOperation);
+        }
+    }
+
+    public static class S3ClientFactory
+    {
+        public static List<AmazonS3Client> CreateS3Clients(
+            string serviceUrls,
+            string regions,
+            string accessKeyIds,
+            string secretKeys,
+            bool forcePathStyle = true)
+        {
+            if (string.IsNullOrWhiteSpace(serviceUrls))
+                return new List<AmazonS3Client>();
+
+            var urls = SplitAndTrim(serviceUrls);
+            var regionList = SplitAndTrim(regions);
+            var keyIds = SplitAndTrim(accessKeyIds);
+            var secrets = SplitAndTrim(secretKeys);
+
+            if (urls.Length != regionList.Length ||
+                urls.Length != keyIds.Length ||
+                urls.Length != secrets.Length)
+            {
+                throw new ArgumentException("All S3 configuration parameters must have the same number of values");
+            }
+
+            var clients = new List<AmazonS3Client>();
+
+            for (int i = 0; i < urls.Length; i++)
+            {
+                var url = urls[i];
+                if (string.IsNullOrWhiteSpace(url))
+                    continue;
+
+                // Get corresponding values from other arrays
+                var region = regionList[i];
+                var keyId = keyIds[i];
+                var secret = secrets[i];
+
+                // Validate that all required values are present
+                if (string.IsNullOrWhiteSpace(region) ||
+                    string.IsNullOrWhiteSpace(keyId) ||
+                    string.IsNullOrWhiteSpace(secret))
+                {
+                    System.Diagnostics.Debug.WriteLine(
+                        $"Skipping S3 client at index {i}: Missing required values (URL={url}, Region={region}, KeyId={keyId?.Length > 0}, Secret={secret?.Length > 0})");
+                    continue;
+                }
+
+                try
+                {
+                    var config = new AmazonS3Config
+                    {
+                        ServiceURL = url,
+                        AuthenticationRegion = region,
+                        ForcePathStyle = forcePathStyle,
+                        SignatureMethod = Amazon.Runtime.SigningAlgorithm.HmacSHA256,
+                        SignatureVersion = "4"
+                    };
+
+                    var client = new AmazonS3Client(keyId, secret, config);
+                    clients.Add(client);
+                }
+                catch (Exception ex)
+                {
+                    // Log configuration error but continue with other clients
+                    System.Diagnostics.Debug.WriteLine($"Failed to create S3 client for URL {url}: {ex.Message}");
+                }
+            }
+
+            return clients;
+        }
+
+        private static string[] SplitAndTrim(string? input)
+        {
+            if (string.IsNullOrWhiteSpace(input))
+                return Array.Empty<string>();
+
+            return input.Split(';', StringSplitOptions.None)
+                       .Select(s => s.Trim())
+                       .ToArray();
+        }
+    }
+}

Notesnook.API/Helpers/StorageHelper.cs

@@ -39,11 +39,11 @@ namespace Notesnook.API.Helpers
             return MAX_FILE_SIZE[subscription.Plan];
         }
 
-        public static bool IsStorageLimitReached(Subscription subscription, Limit limit)
+        public static bool IsStorageLimitReached(Subscription subscription, long limit)
         {
             var storageLimit = GetStorageLimitForPlan(subscription);
             if (storageLimit == -1) return false;
-            return limit.Value > storageLimit;
+            return limit > storageLimit;
         }
 
         public static bool IsFileSizeExceeded(Subscription subscription, long fileSize)
@@ -52,6 +52,17 @@ namespace Notesnook.API.Helpers
             return fileSize > maxFileSize;
         }
 
+        public static Limit RolloverStorageLimit(Limit? limit)
+        {
+            var updatedAt = DateTimeOffset.FromUnixTimeMilliseconds(limit?.UpdatedAt ?? 0);
+            if (limit == null || DateTimeOffset.UtcNow.Year > updatedAt.Year || DateTimeOffset.UtcNow.Month > updatedAt.Month)
+            {
+                limit = new Limit { UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), Value = 0 };
+                return limit;
+            }
+            return limit;
+        }
+
         private static readonly string[] sizes = ["B", "KB", "MB", "GB", "TB"];
         public static string FormatBytes(long size)
         {

Notesnook.API/Hubs/SyncHub.cs

@@ -1,473 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Security.Claims;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.SignalR;
-using MongoDB.Driver;
-using Notesnook.API.Authorization;
-using Notesnook.API.Interfaces;
-using Notesnook.API.Models;
-using Notesnook.API.Repositories;
-using Streetwriters.Common.Models;
-using Streetwriters.Data.Interfaces;
-
-namespace Notesnook.API.Hubs
-{
-    public struct RunningPush
-    {
-        public long Timestamp { get; set; }
-        public long Validity { get; set; }
-        public string ConnectionId { get; set; }
-    }
-    public interface ISyncHubClient
-    {
-        Task PushItems(SyncTransferItemV2 transferItem);
-        Task<bool> SendItems(SyncTransferItemV2 transferItem);
-        Task PushCompleted(long lastSynced);
-    }
-
-    public class GlobalSync
-    {
-        private const long PUSH_VALIDITY_EXTENSION_PERIOD = 16 * 1000; // 16 second
-        private const int PUSH_VALIDITY_PERIOD_PER_ITEM = 5 * 100; // 0.5 second
-        private const long BASE_PUSH_VALIDITY_PERIOD = 5 * 1000; // 5 seconds
-        private const long BASE_PUSH_VALIDITY_PERIOD_NEW = 16 * 1000; // 16 seconds
-        private readonly static Dictionary<string, List<RunningPush>> PushOperations = new();
-
-        public static void ClearPushOperations(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations.ToArray())
-                    if (push.ConnectionId == connectionId || !IsPushValid(push, now))
-                        operations.Remove(push);
-            }
-        }
-
-        public static bool IsPushing(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (push.ConnectionId == connectionId && IsPushValid(push, now)) return true;
-            }
-            return false;
-        }
-
-
-        public static bool IsUserPushing(string userId)
-        {
-            var count = 0;
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (IsPushValid(push, now)) ++count;
-            }
-            return count > 0;
-        }
-
-        public static void StartPush(string userId, string connectionId, long? totalItems = null)
-        {
-            if (IsPushing(userId, connectionId)) return;
-
-            if (!PushOperations.ContainsKey(userId))
-                PushOperations[userId] = new List<RunningPush>();
-
-            PushOperations[userId].Add(new RunningPush
-            {
-                ConnectionId = connectionId,
-                Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-                Validity = totalItems.HasValue ? BASE_PUSH_VALIDITY_PERIOD + (totalItems.Value * PUSH_VALIDITY_PERIOD_PER_ITEM) : BASE_PUSH_VALIDITY_PERIOD_NEW
-            });
-        }
-        public static void ExtendPush(string userId, string connectionId)
-        {
-            if (!IsPushing(userId, connectionId) || !PushOperations.ContainsKey(userId))
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var index = PushOperations[userId].FindIndex((push) => push.ConnectionId == connectionId);
-            if (index < 0)
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var pushOperation = PushOperations[userId][index];
-            pushOperation.Validity += PUSH_VALIDITY_EXTENSION_PERIOD;
-        }
-        private static bool IsPushValid(RunningPush push, long now)
-        {
-            return now < push.Timestamp + push.Validity;
-        }
-    }
-
-    [Authorize("Sync")]
-    public class SyncHub : Hub<ISyncHubClient>
-    {
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = new[] {
-            "settings",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "relation", // relations must sync at the end to prevent invalid state
-        };
-
-        public SyncHub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
-        {
-            Repositories = syncItemsRepositoryAccessor;
-            unit = unitOfWork;
-        }
-
-        public override async Task OnConnectedAsync()
-        {
-            var result = new SyncRequirement().IsAuthorized(Context.User, new PathString("/hubs/sync"));
-            if (!result.Succeeded)
-            {
-                var reason = result.AuthorizationFailure.FailureReasons.FirstOrDefault();
-                throw new HubException(reason?.Message ?? "Unauthorized");
-            }
-            var id = Context.User.FindFirstValue("sub");
-            await Groups.AddToGroupAsync(Context.ConnectionId, id);
-            await base.OnConnectedAsync();
-        }
-
-        public override async Task OnDisconnectedAsync(Exception exception)
-        {
-            try
-            {
-                await base.OnDisconnectedAsync(exception);
-            }
-            finally
-            {
-                var id = Context.User.FindFirstValue("sub");
-                GlobalSync.ClearPushOperations(id, Context.ConnectionId);
-            }
-        }
-
-        private Action<SyncItem, string, long> MapTypeToUpsertAction(string type)
-        {
-            return type switch
-            {
-                "attachment" => Repositories.Attachments.Upsert,
-                "note" => Repositories.Notes.Upsert,
-                "notebook" => Repositories.Notebooks.Upsert,
-                "content" => Repositories.Contents.Upsert,
-                "shortcut" => Repositories.Shortcuts.Upsert,
-                "reminder" => Repositories.Reminders.Upsert,
-                "relation" => Repositories.Relations.Upsert,
-                _ => null,
-            };
-        }
-
-        public async Task<long> InitializePush(SyncMetadata syncMetadata)
-        {
-            if (syncMetadata.LastSynced <= 0) throw new HubException("Last synced time cannot be zero or less than zero.");
-
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) return 0;
-
-            UserSettings userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            long dateSynced = Math.Max(syncMetadata.LastSynced, userSettings.LastSynced);
-
-            GlobalSync.StartPush(userId, Context.ConnectionId);
-
-
-            if (
-                (userSettings.VaultKey != null &&
-                syncMetadata.VaultKey != null &&
-                !userSettings.VaultKey.Equals(syncMetadata.VaultKey) &&
-                !syncMetadata.VaultKey.IsEmpty()) ||
-                (userSettings.VaultKey == null &&
-                syncMetadata.VaultKey != null &&
-                !syncMetadata.VaultKey.IsEmpty()))
-            {
-                userSettings.VaultKey = syncMetadata.VaultKey;
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-            }
-
-            return dateSynced;
-        }
-
-        public async Task<int> PushItems(SyncTransferItemV2 pushItem, long dateSynced)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) return 0;
-
-            SyncEventCounterSource.Log.Push();
-
-            try
-            {
-                var others = Clients.OthersInGroup(userId);
-                others.PushItems(pushItem);
-
-                GlobalSync.ExtendPush(userId, Context.ConnectionId);
-
-                if (pushItem.Type == "settings")
-                {
-                    var settings = pushItem.Items.First();
-                    if (settings == null) return 0;
-                    settings.Id = MongoDB.Bson.ObjectId.Parse(userId);
-                    settings.ItemId = userId;
-                    Repositories.LegacySettings.Upsert(settings, userId, dateSynced);
-                }
-                else
-                {
-                    var UpsertItem = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception("Invalid item type.");
-                    foreach (var item in pushItem.Items)
-                    {
-                        UpsertItem(item, userId, dateSynced);
-                    }
-                }
-
-                return await unit.Commit() ? 1 : 0;
-            }
-            catch (Exception ex)
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-                throw ex;
-            }
-        }
-
-        public async Task<bool> SyncCompleted(long dateSynced)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            try
-            {
-                UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-
-                long lastSynced = Math.Max(dateSynced, userSettings.LastSynced);
-
-                userSettings.LastSynced = lastSynced;
-
-                await this.Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-
-                await Clients.OthersInGroup(userId).PushCompleted(lastSynced);
-
-                return true;
-            }
-            finally
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-            }
-        }
-
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, long, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, long lastSyncedTimestamp, int size, long maxBytes, int skipChunks)
-        {
-            var chunksProcessed = 0;
-            for (int i = 0; i < collections.Length; i++)
-            {
-                var type = types[i];
-
-                using var cursor = await collections[i](userId, lastSyncedTimestamp, size);
-
-                var chunk = new List<SyncItem>();
-                long totalBytes = 0;
-                long METADATA_BYTES = 5 * 1024;
-
-                while (await cursor.MoveNextAsync())
-                {
-                    if (chunksProcessed++ < skipChunks) continue;
-                    foreach (var item in cursor.Current)
-                    {
-                        chunk.Add(item);
-                        totalBytes += item.Length + METADATA_BYTES;
-                        if (totalBytes >= maxBytes)
-                        {
-                            yield return new SyncTransferItemV2
-                            {
-                                Items = chunk,
-                                Type = type,
-                                Count = chunksProcessed
-                            };
-
-                            totalBytes = 0;
-                            chunk.Clear();
-                        }
-                    }
-                }
-                if (chunk.Count > 0)
-                {
-                    if (chunksProcessed++ < skipChunks) continue;
-                    yield return new SyncTransferItemV2
-                    {
-                        Items = chunk,
-                        Type = type,
-                        Count = chunksProcessed
-                    };
-                }
-            }
-        }
-
-        public Task<SyncMetadata> RequestFetch(long lastSyncedTimestamp)
-        {
-            return RequestResumableFetch(lastSyncedTimestamp);
-        }
-
-        public async Task<SyncMetadata> RequestResumableFetch(long lastSyncedTimestamp, int cursor = 0)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-
-            if (GlobalSync.IsUserPushing(userId))
-            {
-                throw new HubException("Cannot fetch data while another sync is in progress. Please try again later.");
-            }
-
-            SyncEventCounterSource.Log.Fetch();
-
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
-            {
-                throw new HubException($"Provided timestamp value is too large. Server timestamp: {userSettings.LastSynced} Sent timestamp: {lastSyncedTimestamp}. Please run a Force Sync to fix this issue.");
-            }
-            // var client = Clients.Caller;
-
-            if (lastSyncedTimestamp > 0 && userSettings.LastSynced == lastSyncedTimestamp)
-            {
-                return new SyncMetadata
-                {
-                    LastSynced = userSettings.LastSynced,
-                };
-            }
-
-            var isResumable = lastSyncedTimestamp == 0;
-            if (!isResumable) cursor = 0;
-
-            var chunks = PrepareChunks(
-                collections: new[] {
-                    Repositories.LegacySettings.FindItemsSyncedAfter,
-                    Repositories.Attachments.FindItemsSyncedAfter,
-                    Repositories.Notes.FindItemsSyncedAfter,
-                    Repositories.Notebooks.FindItemsSyncedAfter,
-                    Repositories.Contents.FindItemsSyncedAfter,
-                    Repositories.Shortcuts.FindItemsSyncedAfter,
-                    Repositories.Reminders.FindItemsSyncedAfter,
-                    Repositories.Relations.FindItemsSyncedAfter,
-                },
-                types: CollectionKeys,
-                userId,
-                lastSyncedTimestamp,
-                size: 1000,
-                maxBytes: 7 * 1024 * 1024,
-                skipChunks: cursor
-            );
-
-            await foreach (var chunk in chunks)
-            {
-                _ = await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10));
-            }
-
-            return new SyncMetadata
-            {
-                VaultKey = userSettings.VaultKey,
-                LastSynced = userSettings.LastSynced,
-            };
-        }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct BatchedSyncTransferItem
-    {
-        [MessagePack.Key("lastSynced")]
-        public long LastSynced { get; set; }
-
-        [MessagePack.Key("items")]
-        public string[] Items { get; set; }
-
-        [MessagePack.Key("types")]
-        public string[] Types { get; set; }
-
-        [MessagePack.Key("total")]
-        public int Total { get; set; }
-
-        [MessagePack.Key("current")]
-        public int Current { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncTransferItem
-    {
-        [MessagePack.Key("synced")]
-        public bool Synced { get; set; }
-
-        [MessagePack.Key("lastSynced")]
-        public long LastSynced { get; set; }
-
-        [MessagePack.Key("item")]
-        public string Item { get; set; }
-
-        [MessagePack.Key("itemType")]
-        public string ItemType { get; set; }
-
-        [MessagePack.Key("total")]
-        public int Total { get; set; }
-
-        [MessagePack.Key("current")]
-        public int Current { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncTransferItemV2
-    {
-        [MessagePack.Key("items")]
-        [JsonPropertyName("items")]
-        public IEnumerable<SyncItem> Items { get; set; }
-
-        [MessagePack.Key("type")]
-        [JsonPropertyName("type")]
-        public string Type { get; set; }
-        [MessagePack.Key("count")]
-        [JsonPropertyName("count")]
-        public int Count { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncMetadata
-    {
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
-
-        [MessagePack.Key("lastSynced")]
-        [JsonPropertyName("lastSynced")]
-        public long LastSynced { get; set; }
-        // [MessagePack.Key("total")]
-        // public long TotalItems { get; set; }
-    }
-}

Notesnook.API/Hubs/SyncV2Hub.cs

@@ -29,6 +29,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using Notesnook.API.Authorization;
 using Notesnook.API.Interfaces;
@@ -45,12 +46,14 @@ namespace Notesnook.API.Hubs
         Task<bool> SendMonographs(IEnumerable<MonographMetadata> monographs);
         Task<bool> SendInboxItems(IEnumerable<InboxSyncItem> inboxItems);
         Task PushCompleted();
+        Task PushCompletedV2(string deviceId);
     }
 
     [Authorize]
     public class SyncV2Hub : Hub<ISyncV2HubClient>
     {
         private ISyncItemsRepositoryAccessor Repositories { get; }
+        private SyncDeviceService SyncDeviceService { get; }
         private readonly IUnitOfWork unit;
         private static readonly string[] CollectionKeys = [
             "settingitem",
@@ -66,12 +69,15 @@ namespace Notesnook.API.Hubs
             "relation", // relations must sync at the end to prevent invalid state
         ];
         private readonly FrozenDictionary<string, Action<IEnumerable<SyncItem>, string, long>> UpsertActionsMap;
-        private readonly Func<string, string[], bool, int, Task<IAsyncCursor<SyncItem>>>[] Collections;
+        private readonly Func<string, IEnumerable<string>, bool, int, Task<IAsyncCursor<SyncItem>>>[] Collections;
+        ILogger<SyncV2Hub> Logger { get; }
 
-        public SyncV2Hub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
+        public SyncV2Hub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork, SyncDeviceService syncDeviceService, ILogger<SyncV2Hub> logger)
         {
+            Logger = logger;
             Repositories = syncItemsRepositoryAccessor;
             unit = unitOfWork;
+            SyncDeviceService = syncDeviceService;
 
             Collections = [
                 Repositories.Settings.FindItemsById,
@@ -130,7 +136,7 @@ namespace Notesnook.API.Hubs
 
                 if (!await unit.Commit()) return 0;
 
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).AddIdsToOtherDevices(pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
+                await SyncDeviceService.AddIdsToOtherDevicesAsync(userId, deviceId, pushItem.Items.Select((i) => new ItemKey(i.ItemId, pushItem.Type)));
                 return 1;
             }
             finally
@@ -146,14 +152,22 @@ namespace Notesnook.API.Hubs
             return true;
         }
 
-        private async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(string userId, string[] ids, int size, bool resetSync, long maxBytes)
+        public async Task<bool> PushCompletedV2(string deviceId)
+        {
+            var userId = Context.User?.FindFirstValue("sub") ?? throw new HubException("User not found.");
+            await Clients.OthersInGroup(userId).PushCompleted();
+            await Clients.OthersInGroup(userId).PushCompletedV2(deviceId);
+            return true;
+        }
+
+        private async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(string userId, HashSet<ItemKey> ids, int size, bool resetSync, long maxBytes)
         {
             var itemsProcessed = 0;
             for (int i = 0; i < Collections.Length; i++)
             {
                 var type = CollectionKeys[i];
 
-                var filteredIds = ids.Where((id) => id.EndsWith($":{type}")).Select((id) => id.Split(":")[0]).ToArray();
+                var filteredIds = ids.Where((id) => id.Type == type).Select((id) => id.ItemId).ToArray();
                 if (!resetSync && filteredIds.Length == 0) continue;
 
                 using var cursor = await Collections[i](userId, filteredIds, resetSync, size);
@@ -217,61 +231,47 @@ namespace Notesnook.API.Hubs
 
             SyncEventCounterSource.Log.FetchV2();
 
-            var device = new SyncDevice(userId, deviceId);
-            var deviceService = new SyncDeviceService(device);
-            if (!deviceService.IsDeviceRegistered()) deviceService.RegisterDevice();
-
-            device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-
-            var isResetSync = deviceService.IsSyncReset();
-            if (!deviceService.IsUnsynced() &&
-                !deviceService.IsSyncPending() &&
-                !isResetSync)
-                return new SyncV2Metadata { Synced = true };
+            var device = await SyncDeviceService.GetDeviceAsync(userId, deviceId);
+            if (device == null)
+                device = await SyncDeviceService.RegisterDeviceAsync(userId, deviceId);
+            else
+                await SyncDeviceService.UpdateLastAccessTimeAsync(userId, deviceId);
 
             var stopwatch = Stopwatch.StartNew();
             try
             {
-                string[] ids = deviceService.FetchUnsyncedIds();
+                var ids = await SyncDeviceService.FetchUnsyncedIdsAsync(userId, deviceId);
+                if (!device.IsSyncReset && ids.Count == 0)
+                    return new SyncV2Metadata { Synced = true };
 
                 var chunks = PrepareChunks(
                     userId,
                     ids,
-                    size: 1000,
-                    resetSync: isResetSync,
+                    size: 100,
+                    resetSync: device.IsSyncReset,
                     maxBytes: 7 * 1024 * 1024
                 );
 
-                var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId.Equals(userId));
-                if (userSettings.VaultKey != null)
-                {
-                    if (!await Clients.Caller.SendVaultKey(userSettings.VaultKey).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected vault key.");
-                }
-
-
                 await foreach (var chunk in chunks)
                 {
                     if (!await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected sent items.");
 
-                    if (!isResetSync)
+                    if (!device.IsSyncReset)
                     {
-                        var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
-                        ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
-                        deviceService.WritePendingIds(ids);
+                        ids.ExceptWith(chunk.Items.Select(i => new ItemKey(i.ItemId, chunk.Type)));
+                        await SyncDeviceService.WritePendingIdsAsync(userId, deviceId, ids);
                     }
                 }
 
                 if (includeMonographs)
                 {
-                    var isSyncingMonographsForFirstTime = !device.HasInitialMonographsSync;
-                    var unsyncedMonographs = ids.Where((id) => id.EndsWith(":monograph")).ToHashSet();
-                    var unsyncedMonographIds = unsyncedMonographs.Select((id) => id.Split(":")[0]).ToArray();
-                    FilterDefinition<Monograph> filter = isResetSync || isSyncingMonographsForFirstTime
-                        ? Builders<Monograph>.Filter.Eq("UserId", userId)
+                    var unsyncedMonographIds = ids.Where(k => k.Type == "monograph").Select(k => k.ItemId);
+                    FilterDefinition<Monograph> filter = device.IsSyncReset
+                        ? Builders<Monograph>.Filter.Eq(m => m.UserId, userId)
                         : Builders<Monograph>.Filter.And(
-                            Builders<Monograph>.Filter.Eq("UserId", userId),
+                            Builders<Monograph>.Filter.Eq(m => m.UserId, userId),
                             Builders<Monograph>.Filter.Or(
-                                Builders<Monograph>.Filter.In("ItemId", unsyncedMonographIds),
+                                Builders<Monograph>.Filter.In(m => m.ItemId, unsyncedMonographIds),
                                 Builders<Monograph>.Filter.In("_id", unsyncedMonographIds)
                             )
                         );
@@ -282,29 +282,26 @@ namespace Notesnook.API.Hubs
                         Password = m.Password,
                         SelfDestruct = m.SelfDestruct,
                         Title = m.Title,
-                        ItemId = m.ItemId ?? m.Id.ToString(),
+                        ItemId = m.ItemId ?? m.Id.ToString()
                     }).ToListAsync();
 
                     if (userMonographs.Count > 0 && !await Clients.Caller.SendMonographs(userMonographs).WaitAsync(TimeSpan.FromMinutes(10)))
                         throw new HubException("Client rejected monographs.");
-
-                    device.HasInitialMonographsSync = true;
                 }
 
                 if (includeInboxItems)
                 {
-                    var unsyncedInboxItems = ids.Where((id) => id.EndsWith(":inboxItems")).ToHashSet();
-                    var unsyncedInboxItemIds = unsyncedInboxItems.Select((id) => id.Split(":")[0]).ToArray();
-                    var userInboxItems = isResetSync
+                    var unsyncedInboxItemIds = ids.Where(k => k.Type == "inbox_item").Select(k => k.ItemId);
+                    var userInboxItems = device.IsSyncReset
                         ? await Repositories.InboxItems.FindAsync(m => m.UserId == userId)
-                        : await Repositories.InboxItems.FindAsync(m => m.UserId == userId && unsyncedInboxItemIds.Contains(m.ItemId));
+                        : await Repositories.InboxItems.FindAsync(m => m.UserId == userId && unsyncedInboxItemIds.Contains(m.ItemId ?? m.Id.ToString()));
                     if (userInboxItems.Any() && !await Clients.Caller.SendInboxItems(userInboxItems).WaitAsync(TimeSpan.FromMinutes(10)))
                     {
                         throw new HubException("Client rejected inbox items.");
                     }
                 }
 
-                deviceService.Reset();
+                await SyncDeviceService.ResetAsync(userId, deviceId);
 
                 return new SyncV2Metadata
                 {
@@ -325,4 +322,19 @@ namespace Notesnook.API.Hubs
         [JsonPropertyName("synced")]
         public bool Synced { get; set; }
     }
+
+    [MessagePack.MessagePackObject]
+    public struct SyncTransferItemV2
+    {
+        [MessagePack.Key("items")]
+        [JsonPropertyName("items")]
+        public IEnumerable<SyncItem> Items { get; set; }
+
+        [MessagePack.Key("type")]
+        [JsonPropertyName("type")]
+        public string Type { get; set; }
+        [MessagePack.Key("count")]
+        [JsonPropertyName("count")]
+        public int Count { get; set; }
+    }
 }

Notesnook.API/Interfaces/IMonograph.cs

@@ -1,33 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using Notesnook.API.Models;
-using Streetwriters.Common.Interfaces;
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IMonograph : IDocument
-    {
-        string Title { get; set; }
-        string UserId { get; set; }
-        byte[] CompressedContent { get; set; }
-        EncryptedData EncryptedContent { get; set; }
-        long DatePublished { get; set; }
-    }
-}

Notesnook.API/Interfaces/IS3Service.cs

@@ -17,23 +17,21 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Threading;
 using System.Threading.Tasks;
 using Amazon.S3.Model;
 using Notesnook.API.Models;
-using Notesnook.API.Models.Responses;
-using Streetwriters.Common.Interfaces;
 
 namespace Notesnook.API.Interfaces
 {
     public interface IS3Service
     {
         Task DeleteObjectAsync(string userId, string name);
+        Task DeleteObjectsAsync(string userId, string[] names);
         Task DeleteDirectoryAsync(string userId);
         Task<long> GetObjectSizeAsync(string userId, string name);
-        string? GetUploadObjectUrl(string userId, string name);
-        string? GetInternalUploadObjectUrl(string userId, string name);
-        Task<string?> GetDownloadObjectUrl(string userId, string name);
+        Task<string?> GetUploadObjectUrlAsync(string userId, string name);
+        Task<string?> GetInternalUploadObjectUrlAsync(string userId, string name);
+        Task<string?> GetDownloadObjectUrlAsync(string userId, string name);
         Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string? uploadId = null);
         Task AbortMultipartUploadAsync(string userId, string name, string uploadId);
         Task CompleteMultipartUploadAsync(string userId, CompleteMultipartUploadRequest uploadRequest);

Notesnook.API/Interfaces/ISyncItemsRepositoryAccessor.cs

@@ -42,5 +42,7 @@ namespace Notesnook.API.Interfaces
         Repository<Monograph> Monographs { get; }
         Repository<InboxApiKey> InboxApiKey { get; }
         Repository<InboxSyncItem> InboxItems { get; }
+        Repository<SyncDevice> SyncDevices { get; }
+        Repository<DeviceIdsChunk> DeviceIdsChunks { get; }
     }
 }

Notesnook.API/Interfaces/IUserService.cs

@@ -27,7 +27,7 @@ namespace Notesnook.API.Interfaces
     {
         Task CreateUserAsync();
         Task DeleteUserAsync(string userId);
-        Task DeleteUserAsync(string userId, string jti, string password);
+        Task DeleteUserAsync(string userId, string? jti, string password);
         Task<bool> ResetUserAsync(string userId, bool removeAttachments);
         Task<UserResponse> GetUserAsync(string userId);
         Task SetUserKeysAsync(string userId, UserKeys keys);

Notesnook.API/Interfaces/IUserSettings.cs

@@ -1,42 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using Notesnook.API.Models;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IUserSettings : IDocument
-    {
-        string UserId { get; set; }
-
-        long LastSynced
-        {
-            get; set;
-        }
-
-        EncryptedData VaultKey
-        {
-            get; set;
-        }
-
-        string Salt { get; set; }
-    }
-}

Notesnook.API/Jobs/DeviceCleanupJob.cs

@@ -1,64 +1,41 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
+using MongoDB.Driver;
+using Notesnook.API.Interfaces;
+using Notesnook.API.Models;
+using Notesnook.API.Services;
 using Quartz;
 
 namespace Notesnook.API.Jobs
 {
-    public class DeviceCleanupJob : IJob
+    public class DeviceCleanupJob(ISyncItemsRepositoryAccessor repositories) : IJob
     {
         public async Task Execute(IJobExecutionContext context)
         {
-            ParallelOptions parallelOptions = new()
+            var cutoffDate = DateTimeOffset.UtcNow.AddMonths(-1).ToUnixTimeMilliseconds();
+            var deviceFilter = Builders<SyncDevice>.Filter.Lt(x => x.LastAccessTime, cutoffDate);
+
+            using var cursor = await repositories.SyncDevices.Collection.Find(deviceFilter, new FindOptions { BatchSize = 1000 })
+                .Project(x => x.DeviceId)
+                .ToCursorAsync();
+
+            var deleteModels = new List<WriteModel<DeviceIdsChunk>>();
+            while (await cursor.MoveNextAsync())
             {
-                MaxDegreeOfParallelism = 100,
-                CancellationToken = context.CancellationToken,
-            };
-            Parallel.ForEach(Directory.EnumerateDirectories("sync"), parallelOptions, (userDir, ct) =>
+                if (!cursor.Current.Any()) continue;
+                deleteModels.Add(new DeleteManyModel<DeviceIdsChunk>(Builders<DeviceIdsChunk>.Filter.In(x => x.DeviceId, cursor.Current)));
+            }
+
+            if (deleteModels.Count > 0)
             {
-                foreach (var device in Directory.EnumerateDirectories(userDir))
-                {
-                    string lastAccessFile = Path.Combine(device, "LastAccessTime");
+                var bulkOptions = new BulkWriteOptions { IsOrdered = false };
+                await repositories.DeviceIdsChunks.Collection.BulkWriteAsync(deleteModels, bulkOptions);
+            }
 
-                    try
-                    {
-                        if (!File.Exists(lastAccessFile))
-                        {
-                            Directory.Delete(device, true);
-                            continue;
-                        }
-
-                        string content = File.ReadAllText(lastAccessFile);
-                        if (!long.TryParse(content, out long lastAccessTime) || lastAccessTime <= 0)
-                        {
-                            Directory.Delete(device, true);
-                            continue;
-                        }
-
-                        DateTimeOffset accessTime;
-                        try
-                        {
-                            accessTime = DateTimeOffset.FromUnixTimeMilliseconds(lastAccessTime);
-                        }
-                        catch (Exception)
-                        {
-                            Directory.Delete(device, true);
-                            continue;
-                        }
-
-                        // If the device hasn't been accessed for more than one month, delete it.
-                        if (accessTime.AddMonths(1) < DateTimeOffset.UtcNow)
-                        {
-                            Directory.Delete(device, true);
-                        }
-                    }
-                    catch (Exception ex)
-                    {
-                        // Log the error and continue processing other directories.
-                        Console.Error.WriteLine($"Error processing device '{device}': {ex.Message}");
-                    }
-                }
-            });
+            await repositories.SyncDevices.Collection.DeleteManyAsync(deviceFilter);
         }
     }
 }

Notesnook.API/Models/Announcement.cs

@@ -40,7 +40,7 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("type")]
         [BsonElement("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("timestamp")]
         [BsonElement("timestamp")]
@@ -48,7 +48,7 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("platforms")]
         [BsonElement("platforms")]
-        public string[] Platforms { get; set; }
+        public required string[] Platforms { get; set; }
 
         [JsonPropertyName("isActive")]
         [BsonElement("isActive")]
@@ -56,7 +56,7 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("userTypes")]
         [BsonElement("userTypes")]
-        public string[] UserTypes { get; set; }
+        public required string[] UserTypes { get; set; }
 
         [JsonPropertyName("appVersion")]
         [BsonElement("appVersion")]
@@ -64,63 +64,63 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("body")]
         [BsonElement("body")]
-        public BodyComponent[] Body { get; set; }
+        public required BodyComponent[] Body { get; set; }
 
         [JsonIgnore]
         [BsonElement("userIds")]
-        public string[] UserIds { get; set; }
+        public string[]? UserIds { get; set; }
 
 
         [Obsolete]
         [JsonPropertyName("title")]
         [DataMember(Name = "title")]
         [BsonElement("title")]
-        public string Title { get; set; }
+        public string? Title { get; set; }
 
         [Obsolete]
         [JsonPropertyName("description")]
         [BsonElement("description")]
-        public string Description { get; set; }
+        public string? Description { get; set; }
 
         [Obsolete]
         [JsonPropertyName("callToActions")]
         [BsonElement("callToActions")]
-        public CallToAction[] CallToActions { get; set; }
+        public CallToAction[]? CallToActions { get; set; }
     }
 
     public class BodyComponent
     {
         [JsonPropertyName("type")]
         [BsonElement("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("platforms")]
         [BsonElement("platforms")]
-        public string[] Platforms { get; set; }
+        public string[]? Platforms { get; set; }
 
         [JsonPropertyName("style")]
         [BsonElement("style")]
-        public Style Style { get; set; }
+        public Style? Style { get; set; }
 
         [JsonPropertyName("src")]
         [BsonElement("src")]
-        public string Src { get; set; }
+        public string? Src { get; set; }
 
         [JsonPropertyName("text")]
         [BsonElement("text")]
-        public string Text { get; set; }
+        public string? Text { get; set; }
 
         [JsonPropertyName("value")]
         [BsonElement("value")]
-        public string Value { get; set; }
+        public string? Value { get; set; }
 
         [JsonPropertyName("items")]
         [BsonElement("items")]
-        public BodyComponent[] Items { get; set; }
+        public BodyComponent[]? Items { get; set; }
 
         [JsonPropertyName("actions")]
         [BsonElement("actions")]
-        public CallToAction[] Actions { get; set; }
+        public required CallToAction[] Actions { get; set; }
     }
 
     public class Style
@@ -135,25 +135,25 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("textAlign")]
         [BsonElement("textAlign")]
-        public string TextAlign { get; set; }
+        public string? TextAlign { get; set; }
     }
 
     public class CallToAction
     {
         [JsonPropertyName("type")]
         [BsonElement("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("platforms")]
         [BsonElement("platforms")]
-        public string[] Platforms { get; set; }
+        public string[]? Platforms { get; set; }
 
         [JsonPropertyName("data")]
         [BsonElement("data")]
-        public string Data { get; set; }
+        public string? Data { get; set; }
 
         [JsonPropertyName("title")]
         [BsonElement("title")]
-        public string Title { get; set; }
+        public string? Title { get; set; }
     }
 }

Notesnook.API/Models/CompleteMultipartUploadRequestWrapper.cs

@@ -5,9 +5,9 @@ namespace Notesnook.API.Models;
 
 public class CompleteMultipartUploadRequestWrapper
 {
-    public string Key { get; set; }
-    public List<PartETagWrapper> PartETags { get; set; }
-    public string UploadId { get; set; }
+    public required string Key { get; set; }
+    public required List<PartETagWrapper> PartETags { get; set; }
+    public required string UploadId { get; set; }
 
     public CompleteMultipartUploadRequest ToRequest()
     {

Notesnook.API/Models/DeleteAccountForm.cs

@@ -5,7 +5,7 @@ namespace Notesnook.API.Models
     public class DeleteAccountForm
     {
         [Required]
-        public string Password
+        public required string Password
         {
             get; set;
         }

Notesnook.API/Models/DeleteBulkRequest.cs

@@ -1,29 +1,25 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IEncrypted
-    {
-        string Cipher { get; set; }
-        string IV { get; set; }
-        long Length { get; set; }
-        string Salt { get; set; }
-    }
-}
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Notesnook.API.Models;
+
+public class DeleteBulkObjectsRequest
+{
+    public required string[] Names { get; set; }
+}

Notesnook.API/Models/DeviceIdsChunk.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Notesnook.API.Models
+{
+    public class DeviceIdsChunk
+    {
+        [BsonId]
+        public ObjectId Id { get; set; }
+        public required string UserId { get; set; }
+        public required string DeviceId { get; set; }
+        public required string Key { get; set; }
+        public required string[] Ids { get; set; } = [];
+    }
+}

Notesnook.API/Models/EncryptedData.cs

@@ -26,25 +26,19 @@ using System.Text.Json.Serialization;
 namespace Notesnook.API.Models
 {
     [MessagePack.MessagePackObject]
-    public class EncryptedData : IEncrypted
+    public class EncryptedData
     {
         [MessagePack.Key("iv")]
         [JsonPropertyName("iv")]
         [BsonElement("iv")]
         [DataMember(Name = "iv")]
-        public string IV
-        {
-            get; set;
-        }
+        public required string IV { get; set; }
 
         [MessagePack.Key("cipher")]
         [JsonPropertyName("cipher")]
         [BsonElement("cipher")]
         [DataMember(Name = "cipher")]
-        public string Cipher
-        {
-            get; set;
-        }
+        public required string Cipher { get; set; }
 
         [MessagePack.Key("length")]
         [JsonPropertyName("length")]
@@ -56,9 +50,9 @@ namespace Notesnook.API.Models
         [JsonPropertyName("salt")]
         [BsonElement("salt")]
         [DataMember(Name = "salt")]
-        public string Salt { get; set; }
+        public required string Salt { get; set; }
 
-        public override bool Equals(object obj)
+        public override bool Equals(object? obj)
         {
             if (obj is EncryptedData encryptedData)
             {

Notesnook.API/Models/InboxApiKey.cs

@@ -37,16 +37,16 @@ namespace Notesnook.API.Models
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
         [MessagePack.IgnoreMember]
-        public string Id { get; set; }
+        public string Id { get; set; } = string.Empty;
 
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("name")]
-        public string Name { get; set; }
+        public required string Name { get; set; }
 
         [JsonPropertyName("key")]
-        public string Key { get; set; }
+        public string Key { get; set; } = string.Empty;
 
         [JsonPropertyName("dateCreated")]
         public long DateCreated { get; set; }

Notesnook.API/Models/InboxSyncItem.cs

@@ -31,10 +31,13 @@ namespace Notesnook.API.Models
         [JsonPropertyName("key")]
         [MessagePack.Key("key")]
         [Required]
-        public EncryptedKey Key
-        {
-            get; set;
-        }
+        public required EncryptedKey Key { get; set; }
+
+        [DataMember(Name = "salt")]
+        [JsonPropertyName("salt")]
+        [MessagePack.Key("salt")]
+        [Required]
+        public required string Salt { get; set; }
     }
 
     [MessagePack.MessagePackObject]
@@ -44,19 +47,13 @@ namespace Notesnook.API.Models
         [JsonPropertyName("alg")]
         [MessagePack.Key("alg")]
         [Required]
-        public string Algorithm
-        {
-            get; set;
-        }
+        public required string Algorithm { get; set; }
 
         [DataMember(Name = "cipher")]
         [JsonPropertyName("cipher")]
         [MessagePack.Key("cipher")]
         [Required]
-        public string Cipher
-        {
-            get; set;
-        }
+        public required string Cipher { get; set; }
 
         [JsonPropertyName("length")]
         [DataMember(Name = "length")]

Notesnook.API/Models/Monograph.cs

@@ -29,15 +29,9 @@ namespace Notesnook.API.Models
         [BsonId]
         [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
-        public string Id
-        {
-            get; set;
-        }
+        public required string Id { get; set; }
 
-        public string ItemId
-        {
-            get; set;
-        }
+        public required string ItemId { get; set; }
     }
 
     public class Monograph
@@ -50,23 +44,17 @@ namespace Notesnook.API.Models
         [DataMember(Name = "id")]
         [JsonPropertyName("id")]
         [MessagePack.Key("id")]
-        public string ItemId
-        {
-            get; set;
-        }
+        public string? ItemId { get; set; }
 
         [BsonId]
         [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
         [MessagePack.IgnoreMember]
-        public string Id
-        {
-            get; set;
-        }
+        public string Id { get; set; } = string.Empty;
 
         [JsonPropertyName("title")]
-        public string Title { get; set; }
+        public string? Title { get; set; }
 
         [JsonPropertyName("userId")]
         public string? UserId { get; set; }
@@ -92,5 +80,8 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("deleted")]
         public bool Deleted { get; set; }
+
+        [JsonPropertyName("viewCount")]
+        public int ViewCount { get; set; }
     }
 }

Notesnook.API/Models/MonographContent.cs

@@ -28,8 +28,8 @@ namespace Notesnook.API.Models
     public class MonographContent
     {
         [JsonPropertyName("data")]
-        public string Data { get; set; }
+        public required string Data { get; set; }
         [JsonPropertyName("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
     }
 }

Notesnook.API/Models/MonographMetadata.cs

@@ -17,10 +17,10 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
-using System.Runtime.Serialization;
 
 namespace Notesnook.API.Models
 {
@@ -35,7 +35,7 @@ namespace Notesnook.API.Models
         }
 
         [JsonPropertyName("title")]
-        public required string Title { get; set; }
+        public string? Title { get; set; }
 
         [JsonPropertyName("selfDestruct")]
         public bool SelfDestruct { get; set; }

Notesnook.API/Models/MultipartUploadMeta.cs

@@ -17,11 +17,13 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
+
 namespace Notesnook.API.Models
 {
     public class MultipartUploadMeta
     {
-        public string UploadId { get; set; }
-        public string[] Parts { get; set; }
+        public string UploadId { get; set; } = string.Empty;
+        public string[] Parts { get; set; } = Array.Empty<string>();
     }
 }

Notesnook.API/Models/PartETagWrapper.cs

@@ -3,5 +3,5 @@
 public class PartETagWrapper
 {
     public int PartNumber { get; set; }
-    public string ETag { get; set; }
+    public string ETag { get; set; } = string.Empty;
 }

Notesnook.API/Models/Responses/SignupResponse.cs

@@ -6,9 +6,9 @@ namespace Notesnook.API.Models.Responses
     public class SignupResponse : Response
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public string? UserId { get; set; }
 
         [JsonPropertyName("errors")]
-        public string[] Errors { get; set; }
+        public string[]? Errors { get; set; }
     }
 }

Notesnook.API/Models/S3Options.cs

@@ -21,9 +21,9 @@ namespace Notesnook.API.Models
 {
     public class S3Options
     {
-        public string ServiceUrl { get; set; }
-        public string Region { get; set; }
-        public string AccessKeyId { get; set; }
-        public string SecretAccessKey { get; set; }
+        public string ServiceUrl { get; set; } = string.Empty;
+        public string Region { get; set; } = string.Empty;
+        public string AccessKeyId { get; set; } = string.Empty;
+        public string SecretAccessKey { get; set; } = string.Empty;
     }
 }

Notesnook.API/Models/SyncDevice.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Notesnook.API.Models
+{
+    public class SyncDevice
+    {
+        [BsonId]
+        public ObjectId Id { get; set; }
+
+        public required string UserId { get; set; }
+        public required string DeviceId { get; set; }
+        public required long LastAccessTime { get; set; }
+        public required bool IsSyncReset { get; set; }
+        public string? AppVersion { get; set; }
+        public string? DatabaseVersion { get; set; }
+    }
+}

Notesnook.API/Models/SyncItem.cs

@@ -53,20 +53,14 @@ namespace Notesnook.API.Models
         [DataMember(Name = "iv")]
         [MessagePack.Key("iv")]
         [Required]
-        public string IV
-        {
-            get; set;
-        }
+        public string IV { get; set; } = string.Empty;
 
 
         [JsonPropertyName("cipher")]
         [DataMember(Name = "cipher")]
         [MessagePack.Key("cipher")]
         [Required]
-        public string Cipher
-        {
-            get; set;
-        }
+        public string Cipher { get; set; } = string.Empty;
 
         [DataMember(Name = "id")]
         [JsonPropertyName("id")]
@@ -108,10 +102,7 @@ namespace Notesnook.API.Models
         [DataMember(Name = "alg")]
         [MessagePack.Key("alg")]
         [Required]
-        public string Algorithm
-        {
-            get; set;
-        }
+        public string Algorithm { get; set; } = string.Empty;
     }
 
     public class SyncItemBsonSerializer : SerializerBase<SyncItem>

Notesnook.API/Models/UserSettings.cs

@@ -17,6 +17,7 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Notesnook.API.Interfaces;
@@ -25,27 +26,40 @@ namespace Notesnook.API.Models
 {
     public class Limit
     {
-        public long Value { get; set; }
-        public long UpdatedAt { get; set; }
+        private long _value = 0;
+        public long Value
+        {
+            get => _value;
+            set
+            {
+                _value = value;
+                UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+            }
+        }
+        public long UpdatedAt
+        {
+            get;
+            set;
+        }
     }
 
-    public class UserSettings : IUserSettings
+    public class UserSettings
     {
         public UserSettings()
         {
-            this.Id = ObjectId.GenerateNewId().ToString();
+            this.Id = ObjectId.GenerateNewId();
         }
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
         public long LastSynced { get; set; }
-        public string Salt { get; set; }
+        public required string Salt { get; set; }
         public EncryptedData? VaultKey { get; set; }
         public EncryptedData? AttachmentsKey { get; set; }
         public EncryptedData? MonographPasswordsKey { get; set; }
         public InboxKeys? InboxKeys { get; set; }
-        public Limit StorageLimit { get; set; }
+        public Limit? StorageLimit { get; set; }
 
         [BsonId]
         [BsonRepresentation(BsonType.ObjectId)]
-        public string Id { get; set; }
+        public ObjectId Id { get; set; }
     }
 }

Notesnook.API/Notesnook.API.csproj

@@ -1,13 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <StartupObject>Notesnook.API.Program</StartupObject>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="AngleSharp" Version="1.3.0" />
+    <PackageReference Include="AspNetCore.HealthChecks.Aws.S3" Version="9.0.0" />
     <PackageReference Include="AWSSDK.Core" Version="3.7.304.31" />
     <PackageReference Include="DotNetEnv" Version="2.3.0" />
     <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="6.2.0" />

Notesnook.API/Program.cs

@@ -17,13 +17,12 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System;
+using System.Net;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Streetwriters.Common;
-using System.Net;
 
 namespace Notesnook.API
 {
@@ -42,6 +41,12 @@ namespace Notesnook.API
 
         public static IHostBuilder CreateHostBuilder(string[] args) =>
             Host.CreateDefaultBuilder(args)
+                .ConfigureLogging(logging =>
+                {
+                    logging.ClearProviders();
+                    logging.AddConsole();
+                    logging.AddSystemdConsole();
+                })
                 .ConfigureWebHostDefaults(webBuilder =>
                 {
                     webBuilder
@@ -50,7 +55,7 @@ namespace Notesnook.API
                     {
                         options.Limits.MaxRequestBodySize = long.MaxValue;
                         options.ListenAnyIP(Servers.NotesnookAPI.Port);
-                        if (Servers.NotesnookAPI.IsSecure)
+                        if (Servers.NotesnookAPI.IsSecure && Servers.NotesnookAPI.SSLCertificate != null)
                         {
                             options.ListenAnyIP(443, listenerOptions =>
                             {

Notesnook.API/Repositories/SyncItemsRepository.cs

@@ -26,6 +26,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using IdentityModel;
 using Microsoft.VisualBasic;
+using Microsoft.Extensions.Logging;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using Notesnook.API.Hubs;
@@ -41,9 +42,11 @@ namespace Notesnook.API.Repositories
     public class SyncItemsRepository : Repository<SyncItem>
     {
         private readonly string collectionName;
-        public SyncItemsRepository(IDbContext dbContext, IMongoCollection<SyncItem> collection) : base(dbContext, collection)
+        private readonly ILogger<SyncItemsRepository> logger;
+        public SyncItemsRepository(IDbContext dbContext, IMongoCollection<SyncItem> collection, ILogger<SyncItemsRepository> logger) : base(dbContext, collection)
         {
             this.collectionName = collection.CollectionNamespace.CollectionName;
+            this.logger = logger;
         }
 
         private readonly List<string> ALGORITHMS = [Algorithms.Default, Algorithms.XSAL_X25519_7];
@@ -88,11 +91,7 @@ namespace Notesnook.API.Repositories
         public void DeleteByUserId(string userId)
         {
             var filter = Builders<SyncItem>.Filter.Eq("UserId", userId);
-            var writes = new List<WriteModel<SyncItem>>
-            {
-                new DeleteManyModel<SyncItem>(filter)
-            };
-            dbContext.AddCommand((handle, ct) => Collection.BulkWriteAsync(handle, writes, options: null, ct));
+            dbContext.AddCommand((handle, ct) => Collection.DeleteManyAsync(handle, filter, null, ct));
         }
 
         public void Upsert(SyncItem item, string userId, long dateSynced)
@@ -110,7 +109,8 @@ namespace Notesnook.API.Repositories
             // Handle case where the cipher is corrupted.
             if (!IsBase64String(item.Cipher))
             {
-                Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
+                logger.LogError("Corrupted item {ItemId} in collection {CollectionName}. Length: {Length}, Cipher: {Cipher}",
+                    item.ItemId, this.collectionName, item.Length, item.Cipher);
                 throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
             }
 
@@ -147,7 +147,8 @@ namespace Notesnook.API.Repositories
                 // Handle case where the cipher is corrupted.
                 if (!IsBase64String(item.Cipher))
                 {
-                    Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
+                    logger.LogError("Corrupted item {ItemId} in collection {CollectionName}. Length: {Length}, Cipher: {Cipher}",
+                        item.ItemId, this.collectionName, item.Length, item.Cipher);
                     throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
                 }
 

Notesnook.API/Services/S3Service.cs

@@ -24,17 +24,15 @@ using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Amazon;
-using Amazon.Runtime;
 using Amazon.S3;
 using Amazon.S3.Model;
-using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
 using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Streetwriters.Common;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
+using Streetwriters.Common.Accessors;
 
 namespace Notesnook.API.Services
 {
@@ -46,10 +44,11 @@ namespace Notesnook.API.Services
 
     public class S3Service : IS3Service
     {
-        private readonly string BUCKET_NAME = Constants.S3_BUCKET_NAME ?? "";
-        private readonly string INTERNAL_BUCKET_NAME = Constants.S3_INTERNAL_BUCKET_NAME ?? "";
-        private AmazonS3Client S3Client { get; }
+        private readonly string BUCKET_NAME = Constants.S3_BUCKET_NAME;
+        private readonly string INTERNAL_BUCKET_NAME = Constants.S3_INTERNAL_BUCKET_NAME ?? Constants.S3_BUCKET_NAME;
+        private readonly S3FailoverHelper S3Client;
         private ISyncItemsRepositoryAccessor Repositories { get; }
+        private WampServiceAccessor ServiceAccessor { get; }
 
         // When running in a dockerized environment the sync server doesn't have access
         // to the host's S3 Service URL. It can only talk to S3 server via its own internal
@@ -59,61 +58,121 @@ namespace Notesnook.API.Services
         // URLs generated by S3 are host specific. Changing their hostname on the fly causes
         // SignatureDoesNotMatch error.
         // That is why we create 2 separate S3 clients. One for internal traffic and one for external.
-        private AmazonS3Client S3InternalClient { get; }
-        private HttpClient httpClient = new HttpClient();
+        private readonly S3FailoverHelper S3InternalClient;
+        private readonly HttpClient httpClient = new();
 
-        public S3Service(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor)
+        public S3Service(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, WampServiceAccessor wampServiceAccessor, ILogger<S3Service> logger)
         {
             Repositories = syncItemsRepositoryAccessor;
-            var config = new AmazonS3Config
-            {
-#if (DEBUG || STAGING)
-                ServiceURL = Servers.S3Server.ToString(),
-#else
-                ServiceURL = Constants.S3_SERVICE_URL,
-                AuthenticationRegion = Constants.S3_REGION,
-#endif
-                ForcePathStyle = true,
-                SignatureMethod = SigningAlgorithm.HmacSHA256,
-                SignatureVersion = "4"
-            };
-#if (DEBUG || STAGING)
-            S3Client = new AmazonS3Client("S3RVER", "S3RVER", config);
-#else
-            S3Client = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, config);
-#endif
+            ServiceAccessor = wampServiceAccessor;
+            S3Client = new S3FailoverHelper(
+                S3ClientFactory.CreateS3Clients(
+                    Constants.S3_SERVICE_URL,
+                    Constants.S3_REGION,
+                    Constants.S3_ACCESS_KEY_ID,
+                    Constants.S3_ACCESS_KEY,
+                    forcePathStyle: true
+                ),
+                logger: logger
+            );
 
-            if (!string.IsNullOrEmpty(Constants.S3_INTERNAL_SERVICE_URL))
+            if (!string.IsNullOrEmpty(Constants.S3_INTERNAL_SERVICE_URL) && !string.IsNullOrEmpty(Constants.S3_INTERNAL_BUCKET_NAME))
             {
-                S3InternalClient = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, new AmazonS3Config
-                {
-                    ServiceURL = Constants.S3_INTERNAL_SERVICE_URL,
-                    AuthenticationRegion = Constants.S3_REGION,
-                    ForcePathStyle = true,
-                    SignatureMethod = SigningAlgorithm.HmacSHA256,
-                    SignatureVersion = "4"
-                });
+                S3InternalClient = new S3FailoverHelper(
+                    S3ClientFactory.CreateS3Clients(
+                        Constants.S3_INTERNAL_SERVICE_URL,
+                        Constants.S3_REGION,
+                        Constants.S3_ACCESS_KEY_ID,
+                        Constants.S3_ACCESS_KEY,
+                        forcePathStyle: true
+                    ),
+                    logger: logger
+                );
             }
+            else S3InternalClient = S3Client;
 
             AWSConfigsS3.UseSignatureVersion4 = true;
         }
 
         public async Task DeleteObjectAsync(string userId, string name)
         {
-            var objectName = GetFullObjectName(userId, name);
-            if (objectName == null) throw new Exception("Invalid object name."); ;
+            var objectName = GetFullObjectName(userId, name) ?? throw new Exception("Invalid object name.");
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
+            var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.DeleteObjectAsync(INTERNAL_BUCKET_NAME, objectName), operationName: "DeleteObject", isWriteOperation: true);
 
-            if (!IsSuccessStatusCode(((int)response.HttpStatusCode)))
+            if (!IsSuccessStatusCode((int)response.HttpStatusCode))
                 throw new Exception("Could not delete object.");
         }
 
+        public async Task DeleteObjectsAsync(string userId, string[] names)
+        {
+            var objectsToDelete = new List<KeyVersion>();
+
+            foreach (var name in names)
+            {
+                var objectName = GetFullObjectName(userId, name);
+                if (objectName == null) continue;
+
+                objectsToDelete.Add(new KeyVersion { Key = objectName });
+            }
+
+            if (objectsToDelete.Count == 0)
+            {
+                return;
+            }
+
+            // S3 DeleteObjectsRequest supports max 1000 keys per request
+            var batchSize = 1000;
+            var deleteErrors = new List<DeleteError>();
+            var failedBatches = 0;
+
+            for (int i = 0; i < objectsToDelete.Count; i += batchSize)
+            {
+                var batch = objectsToDelete.Skip(i).Take(batchSize).ToList();
+                var deleteObjectsResponse = await S3InternalClient.ExecuteWithFailoverAsync(
+                    (client) => client.DeleteObjectsAsync(new DeleteObjectsRequest
+                    {
+                        BucketName = INTERNAL_BUCKET_NAME,
+                        Objects = batch,
+                    }),
+                    operationName: "DeleteObjects",
+                    isWriteOperation: true
+                );
+
+                if (!IsSuccessStatusCode((int)deleteObjectsResponse.HttpStatusCode))
+                {
+                    failedBatches++;
+                }
+
+                if (deleteObjectsResponse.DeleteErrors.Count > 0)
+                {
+                    deleteErrors.AddRange(deleteObjectsResponse.DeleteErrors);
+                }
+            }
+
+            if (failedBatches > 0 || deleteErrors.Count > 0)
+            {
+                var errorParts = new List<string>();
+
+                if (failedBatches > 0)
+                {
+                    errorParts.Add($"{failedBatches} batch(es) failed with unsuccessful status code");
+                }
+
+                if (deleteErrors.Count > 0)
+                {
+                    errorParts.Add(string.Join(", ", deleteErrors.Select(e => $"{e.Key}: {e.Message}")));
+                }
+
+                throw new Exception(string.Join("; ", errorParts));
+            }
+        }
+
         public async Task DeleteDirectoryAsync(string userId)
         {
             var request = new ListObjectsV2Request
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = INTERNAL_BUCKET_NAME,
                 Prefix = userId,
             };
 
@@ -121,7 +180,7 @@ namespace Notesnook.API.Services
             var keys = new List<KeyVersion>();
             do
             {
-                response = await GetS3Client(S3ClientMode.INTERNAL).ListObjectsV2Async(request);
+                response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.ListObjectsV2Async(request), operationName: "ListObjectsV2");
                 response.S3Objects.ForEach(obj => keys.Add(new KeyVersion
                 {
                     Key = obj.Key,
@@ -133,12 +192,11 @@ namespace Notesnook.API.Services
 
             if (keys.Count <= 0) return;
 
-            var deleteObjectsResponse = await GetS3Client(S3ClientMode.INTERNAL)
-            .DeleteObjectsAsync(new DeleteObjectsRequest
+            var deleteObjectsResponse = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.DeleteObjectsAsync(new DeleteObjectsRequest
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = INTERNAL_BUCKET_NAME,
                 Objects = keys,
-            });
+            }), operationName: "DeleteObjects", isWriteOperation: true);
 
             if (!IsSuccessStatusCode((int)deleteObjectsResponse.HttpStatusCode))
                 throw new Exception("Could not delete directory.");
@@ -146,7 +204,7 @@ namespace Notesnook.API.Services
 
         public async Task<long> GetObjectSizeAsync(string userId, string name)
         {
-            var url = this.GetPresignedURL(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
+            var url = await this.GetPresignedURLAsync(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
             if (url == null) return 0;
 
             var request = new HttpRequestMessage(HttpMethod.Head, url);
@@ -155,17 +213,17 @@ namespace Notesnook.API.Services
         }
 
 
-        public string? GetUploadObjectUrl(string userId, string name)
+        public async Task<string?> GetUploadObjectUrlAsync(string userId, string name)
         {
-            return this.GetPresignedURL(userId, name, HttpVerb.PUT);
+            return await this.GetPresignedURLAsync(userId, name, HttpVerb.PUT);
         }
 
-        public string? GetInternalUploadObjectUrl(string userId, string name)
+        public async Task<string?> GetInternalUploadObjectUrlAsync(string userId, string name)
         {
-            return this.GetPresignedURL(userId, name, HttpVerb.PUT, S3ClientMode.INTERNAL);
+            return await this.GetPresignedURLAsync(userId, name, HttpVerb.PUT, S3ClientMode.INTERNAL);
         }
 
-        public async Task<string?> GetDownloadObjectUrl(string userId, string name)
+        public async Task<string?> GetDownloadObjectUrlAsync(string userId, string name)
         {
             // var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
             // var subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId);
@@ -177,7 +235,7 @@ namespace Notesnook.API.Services
             //     throw new Exception($"You cannot download files larger than {StorageHelper.FormatBytes(fileSizeLimit)} on this plan.");
             // }
 
-            var url = this.GetPresignedURL(userId, name, HttpVerb.GET);
+            var url = await this.GetPresignedURLAsync(userId, name, HttpVerb.GET);
             if (url == null) return null;
             return url;
         }
@@ -189,8 +247,8 @@ namespace Notesnook.API.Services
 
             if (string.IsNullOrEmpty(uploadId))
             {
-                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
-                if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to initiate multipart upload.");
+                var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.InitiateMultipartUploadAsync(INTERNAL_BUCKET_NAME, objectName), operationName: "InitiateMultipartUpload", isWriteOperation: true);
+                if (!IsSuccessStatusCode((int)response.HttpStatusCode)) throw new Exception("Failed to initiate multipart upload.");
 
                 uploadId = response.UploadId;
             }
@@ -198,7 +256,7 @@ namespace Notesnook.API.Services
             var signedUrls = new string[parts];
             for (var i = 0; i < parts; ++i)
             {
-                signedUrls[i] = GetPresignedURLForUploadPart(objectName, uploadId, i + 1);
+                signedUrls[i] = await GetPresignedURLForUploadPartAsync(objectName, uploadId, i + 1);
             }
 
             return new MultipartUploadMeta
@@ -213,14 +271,14 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName, uploadId);
+            var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.AbortMultipartUploadAsync(INTERNAL_BUCKET_NAME, objectName, uploadId), operationName: "AbortMultipartUpload", isWriteOperation: true);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to abort multipart upload.");
         }
 
         private async Task<long> GetMultipartUploadSizeAsync(string userId, string key, string uploadId)
         {
             var objectName = GetFullObjectName(userId, key);
-            var parts = await GetS3Client(S3ClientMode.INTERNAL).ListPartsAsync(GetBucketName(S3ClientMode.INTERNAL), objectName, uploadId);
+            var parts = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.ListPartsAsync(INTERNAL_BUCKET_NAME, objectName, uploadId), operationName: "ListParts");
             long totalSize = 0;
             foreach (var part in parts.Parts)
             {
@@ -240,11 +298,11 @@ namespace Notesnook.API.Services
                 await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                 throw new Exception("User settings not found.");
             }
+            userSettings.StorageLimit ??= StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
 
             if (!Constants.IS_SELF_HOSTED)
             {
-                var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
-                var subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
+                var subscription = await ServiceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
 
                 long fileSize = await GetMultipartUploadSizeAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                 if (StorageHelper.IsFileSizeExceeded(subscription, fileSize))
@@ -253,9 +311,8 @@ namespace Notesnook.API.Services
                     throw new Exception("Max file size exceeded.");
                 }
 
-                userSettings.StorageLimit ??= new Limit { Value = 0, UpdatedAt = 0 };
                 userSettings.StorageLimit.Value += fileSize;
-                if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit))
+                if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value))
                 {
                     await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
                     throw new Exception("Storage limit reached.");
@@ -263,45 +320,55 @@ namespace Notesnook.API.Services
             }
 
             uploadRequest.Key = objectName;
-            uploadRequest.BucketName = GetBucketName(S3ClientMode.INTERNAL);
-            var response = await GetS3Client(S3ClientMode.INTERNAL).CompleteMultipartUploadAsync(uploadRequest);
-            if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to complete multipart upload.");
+            uploadRequest.BucketName = INTERNAL_BUCKET_NAME;
+            var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.CompleteMultipartUploadAsync(uploadRequest), operationName: "CompleteMultipartUpload", isWriteOperation: true);
+            if (!IsSuccessStatusCode((int)response.HttpStatusCode)) throw new Exception("Failed to complete multipart upload.");
 
             if (!Constants.IS_SELF_HOSTED)
             {
-                userSettings.StorageLimit.UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
                 await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+                await Repositories.UsersSettings.Collection.UpdateOneAsync(
+                    Builders<UserSettings>.Filter.Eq(u => u.UserId, userId),
+                    Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
+                );
             }
         }
 
-        private string? GetPresignedURL(string userId, string name, HttpVerb httpVerb, S3ClientMode mode = S3ClientMode.EXTERNAL)
+        private async Task<string?> GetPresignedURLAsync(string userId, string name, HttpVerb httpVerb, S3ClientMode mode = S3ClientMode.EXTERNAL)
         {
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) return null;
 
-            var client = GetS3Client(mode);
-            var request = new GetPreSignedUrlRequest
+            var client = mode == S3ClientMode.INTERNAL ? S3InternalClient : S3Client;
+            var bucketName = mode == S3ClientMode.INTERNAL ? INTERNAL_BUCKET_NAME : BUCKET_NAME;
+
+            return await client.ExecuteWithFailoverAsync(client =>
             {
-                BucketName = GetBucketName(mode),
-                Expires = System.DateTime.Now.AddHours(1),
-                Verb = httpVerb,
-                Key = objectName,
+                var request = new GetPreSignedUrlRequest
+                {
+                    BucketName = bucketName,
+                    Expires = System.DateTime.Now.AddHours(1),
+                    Verb = httpVerb,
+                    Key = objectName,
 #if (DEBUG || STAGING)
-                Protocol = Protocol.HTTP,
+                    Protocol = Protocol.HTTP,
 #else
-                Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
+                    Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
 #endif
-            };
-            return client.GetPreSignedURL(request);
+                };
+                return client.GetPreSignedURLAsync(request);
+            }, operationName: "GetPreSignedURL");
         }
 
-        private string GetPresignedURLForUploadPart(string objectName, string uploadId, int partNumber, S3ClientMode mode = S3ClientMode.EXTERNAL)
+        private Task<string> GetPresignedURLForUploadPartAsync(string objectName, string uploadId, int partNumber, S3ClientMode mode = S3ClientMode.EXTERNAL)
         {
 
-            var client = GetS3Client(mode);
-            return client.GetPreSignedURL(new GetPreSignedUrlRequest
+            var client = mode == S3ClientMode.INTERNAL ? S3InternalClient : S3Client;
+            var bucketName = mode == S3ClientMode.INTERNAL ? INTERNAL_BUCKET_NAME : BUCKET_NAME;
+
+            return client.ExecuteWithFailoverAsync(c => c.GetPreSignedURLAsync(new GetPreSignedUrlRequest
             {
-                BucketName = GetBucketName(mode),
+                BucketName = bucketName,
                 Expires = System.DateTime.Now.AddHours(1),
                 Verb = HttpVerb.PUT,
                 Key = objectName,
@@ -310,32 +377,20 @@ namespace Notesnook.API.Services
 #if (DEBUG || STAGING)
                 Protocol = Protocol.HTTP,
 #else
-                Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
+                Protocol = c.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
 #endif
-            });
+            }), operationName: "GetPreSignedURL");
         }
 
-        private string GetFullObjectName(string userId, string name)
+        private static string? GetFullObjectName(string userId, string name)
         {
             if (userId == null || !Regex.IsMatch(name, "[0-9a-zA-Z!" + Regex.Escape("-") + "_.*'()]")) return null;
             return $"{userId}/{name}";
         }
 
-        bool IsSuccessStatusCode(int statusCode)
+        static bool IsSuccessStatusCode(int statusCode)
         {
             return ((int)statusCode >= 200) && ((int)statusCode <= 299);
         }
-
-        AmazonS3Client GetS3Client(S3ClientMode mode = S3ClientMode.EXTERNAL)
-        {
-            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return S3InternalClient;
-            return S3Client;
-        }
-
-        string GetBucketName(S3ClientMode mode = S3ClientMode.EXTERNAL)
-        {
-            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return INTERNAL_BUCKET_NAME;
-            return BUCKET_NAME;
-        }
     }
 }

Notesnook.API/Services/SyncDeviceService.cs

@@ -24,220 +24,201 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Notesnook.API.Interfaces;
+using Notesnook.API.Models;
 
 namespace Notesnook.API.Services
 {
-    public struct SyncDevice(string userId, string deviceId)
+    public readonly record struct ItemKey(string ItemId, string Type)
     {
-        public readonly string DeviceId => deviceId;
-        public readonly string UserId => userId;
-
-        public string UserSyncDirectoryPath = CreateFilePath(userId);
-        public string UserDeviceDirectoryPath = CreateFilePath(userId, deviceId);
-        public string PendingIdsFilePath = CreateFilePath(userId, deviceId, "pending");
-        public string UnsyncedIdsFilePath = CreateFilePath(userId, deviceId, "unsynced");
-        public string ResetSyncFilePath = CreateFilePath(userId, deviceId, "reset-sync");
-
-        public readonly long LastAccessTime
-        {
-            get => long.Parse(GetMetadata("LastAccessTime") ?? "0");
-            set => SetMetadata("LastAccessTime", value.ToString());
-        }
-
-        /// <summary>
-        /// Indicates if the monographs have been synced for the first time
-        /// ever on a device.
-        /// </summary>
-        public readonly bool HasInitialMonographsSync
-        {
-            get => !string.IsNullOrEmpty(GetMetadata("HasInitialMonographsSync"));
-            set => SetMetadata("HasInitialMonographsSync", value.ToString());
-        }
-
-        private static string CreateFilePath(string userId, string? deviceId = null, string? metadataKey = null)
-        {
-            return Path.Join("sync", userId, deviceId, metadataKey);
-        }
-
-        private readonly string? GetMetadata(string metadataKey)
-        {
-            var path = CreateFilePath(userId, deviceId, metadataKey);
-            if (!File.Exists(path)) return null;
-            return File.ReadAllText(path);
-        }
-
-        private readonly void SetMetadata(string metadataKey, string value)
-        {
-            try
-            {
-                var path = CreateFilePath(userId, deviceId, metadataKey);
-                File.WriteAllText(path, value);
-            }
-            catch (DirectoryNotFoundException) { }
-        }
+        public override string ToString() => $"{ItemId}:{Type}";
     }
-
-    public class SyncDeviceService(SyncDevice device)
+    public class SyncDeviceService(ISyncItemsRepositoryAccessor repositories, ILogger<SyncDeviceService> logger)
     {
-        public string[] GetUnsyncedIds()
-        {
-            try
-            {
-                return File.ReadAllLines(device.UnsyncedIdsFilePath);
-            }
-            catch { return []; }
-        }
+        private static FilterDefinition<SyncDevice> DeviceFilter(string userId, string deviceId) =>
+            Builders<SyncDevice>.Filter.Eq(x => x.UserId, userId) &
+            Builders<SyncDevice>.Filter.Eq(x => x.DeviceId, deviceId);
+        private static FilterDefinition<DeviceIdsChunk> DeviceIdsChunkFilter(string userId, string deviceId, string key) =>
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.UserId, userId) &
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.DeviceId, deviceId) &
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.Key, key);
 
-        public string[] GetUnsyncedIds(string deviceId)
-        {
-            try
-            {
-                return File.ReadAllLines(Path.Join(device.UserSyncDirectoryPath, deviceId, "unsynced"));
-            }
-            catch { return []; }
-        }
+        private static FilterDefinition<DeviceIdsChunk> DeviceIdsChunkFilter(string userId, string deviceId) =>
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.UserId, userId) &
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.DeviceId, deviceId);
 
-        public string[] FetchUnsyncedIds()
+        private static FilterDefinition<DeviceIdsChunk> DeviceIdsChunkFilter(string userId) =>
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.UserId, userId);
+
+        private static FilterDefinition<SyncDevice> UserFilter(string userId) => Builders<SyncDevice>.Filter.Eq(x => x.UserId, userId);
+
+
+        public async Task<HashSet<ItemKey>> GetIdsAsync(string userId, string deviceId, string key)
         {
-            if (IsSyncReset()) return [];
-            try
+            var cursor = await repositories.DeviceIdsChunks.Collection.FindAsync(DeviceIdsChunkFilter(userId, deviceId, key));
+            var result = new HashSet<ItemKey>();
+            while (await cursor.MoveNextAsync())
             {
-                var unsyncedIds = GetUnsyncedIds();
-                lock (device.DeviceId)
+                foreach (var chunk in cursor.Current)
                 {
-                    if (IsSyncPending())
+                    foreach (var id in chunk.Ids)
                     {
-                        unsyncedIds = unsyncedIds.Union(File.ReadAllLines(device.PendingIdsFilePath)).ToArray();
+                        var parts = id.Split(':', 2);
+                        result.Add(new ItemKey(parts[0], parts[1]));
                     }
-
-                    if (unsyncedIds.Length == 0) return [];
-
-                    File.Delete(device.UnsyncedIdsFilePath);
-                    File.WriteAllLines(device.PendingIdsFilePath, unsyncedIds);
                 }
-                return unsyncedIds;
             }
-            catch
+            return result;
+        }
+
+        const int MaxIdsPerChunk = 25_000;
+        public async Task AppendIdsAsync(string userId, string deviceId, string key, IEnumerable<ItemKey> ids)
+        {
+            var filter = DeviceIdsChunkFilter(userId, deviceId, key) & Builders<DeviceIdsChunk>.Filter.Where(x => x.Ids.Length < MaxIdsPerChunk);
+            var chunk = await repositories.DeviceIdsChunks.Collection.Find(filter).FirstOrDefaultAsync();
+
+            if (chunk != null)
             {
-                return [];
+                var update = Builders<DeviceIdsChunk>.Update.AddToSetEach(x => x.Ids, ids.Select(i => i.ToString()));
+                await repositories.DeviceIdsChunks.Collection.UpdateOneAsync(
+                    Builders<DeviceIdsChunk>.Filter.Eq(x => x.Id, chunk.Id),
+                    update
+                );
             }
-        }
-
-        public void WritePendingIds(IEnumerable<string> ids)
-        {
-            lock (device.DeviceId)
+            else
             {
-                File.WriteAllLines(device.PendingIdsFilePath, ids);
-            }
-        }
-
-        public bool IsSyncReset()
-        {
-            return File.Exists(device.ResetSyncFilePath);
-        }
-        public bool IsSyncReset(string deviceId)
-        {
-            return File.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId, "reset-sync"));
-        }
-
-        public bool IsSyncPending()
-        {
-            return File.Exists(device.PendingIdsFilePath);
-        }
-
-        public bool IsUnsynced()
-        {
-            return File.Exists(device.UnsyncedIdsFilePath);
-        }
-
-        public void Reset()
-        {
-            try
-            {
-                lock (device.UserId)
+                var newChunk = new DeviceIdsChunk
                 {
-                    File.Delete(device.ResetSyncFilePath);
-                    File.Delete(device.PendingIdsFilePath);
-                }
+                    UserId = userId,
+                    DeviceId = deviceId,
+                    Key = key,
+                    Ids = [.. ids.Select(i => i.ToString())]
+                };
+                await repositories.DeviceIdsChunks.Collection.InsertOneAsync(newChunk);
             }
-            catch (FileNotFoundException) { }
-            catch (DirectoryNotFoundException) { }
+
+            var emptyChunksFilter = DeviceIdsChunkFilter(userId, deviceId, key) & Builders<DeviceIdsChunk>.Filter.Size(x => x.Ids, 0);
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(emptyChunksFilter);
         }
 
-        public bool IsDeviceRegistered()
+        public async Task WriteIdsAsync(string userId, string deviceId, string key, IEnumerable<ItemKey> ids)
         {
-            return Directory.Exists(device.UserDeviceDirectoryPath);
-        }
-        public bool IsDeviceRegistered(string deviceId)
-        {
-            return Directory.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId));
-        }
-
-        public string[] ListDevices()
-        {
-            return Directory.GetDirectories(device.UserSyncDirectoryPath).Select((path) => path[(path.LastIndexOf(Path.DirectorySeparatorChar) + 1)..]).ToArray();
-        }
-
-        public void ResetDevices()
-        {
-            lock (device.UserId)
+            var writes = new List<WriteModel<DeviceIdsChunk>>
             {
-                if (File.Exists(device.UserSyncDirectoryPath)) File.Delete(device.UserSyncDirectoryPath);
-                Directory.CreateDirectory(device.UserSyncDirectoryPath);
-            }
-        }
-
-        public void AddIdsToOtherDevices(List<string> ids)
-        {
-            device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-            foreach (string id in ListDevices())
+                new DeleteManyModel<DeviceIdsChunk>(DeviceIdsChunkFilter(userId, deviceId, key))
+            };
+            var chunks = ids.Chunk(MaxIdsPerChunk);
+            foreach (var chunk in chunks)
             {
-                if (id == device.DeviceId || IsSyncReset(id)) continue;
-
-                lock (id)
+                var newChunk = new DeviceIdsChunk
                 {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = GetUnsyncedIds(id);
-                    File.WriteAllLines(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds));
-                }
+                    UserId = userId,
+                    DeviceId = deviceId,
+                    Key = key,
+                    Ids = [.. chunk.Select(i => i.ToString())]
+                };
+                writes.Add(new InsertOneModel<DeviceIdsChunk>(newChunk));
             }
+            await repositories.DeviceIdsChunks.Collection.BulkWriteAsync(writes);
         }
 
-        public void AddIdsToAllDevices(List<string> ids)
+        public async Task<HashSet<ItemKey>> FetchUnsyncedIdsAsync(string userId, string deviceId)
         {
-            foreach (var id in ListDevices())
+            var device = await GetDeviceAsync(userId, deviceId);
+            if (device == null || device.IsSyncReset) return [];
+
+            var unsyncedIds = await GetIdsAsync(userId, deviceId, "unsynced");
+            var pendingIds = await GetIdsAsync(userId, deviceId, "pending");
+
+            unsyncedIds = [.. unsyncedIds, .. pendingIds];
+
+            if (unsyncedIds.Count == 0) return [];
+
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId, deviceId, "unsynced"));
+            await WriteIdsAsync(userId, deviceId, "pending", unsyncedIds);
+
+            return unsyncedIds;
+        }
+
+        public async Task WritePendingIdsAsync(string userId, string deviceId, HashSet<ItemKey> ids)
+        {
+            await WriteIdsAsync(userId, deviceId, "pending", ids);
+        }
+
+        public async Task ResetAsync(string userId, string deviceId)
+        {
+            await repositories.SyncDevices.Collection.UpdateOneAsync(DeviceFilter(userId, deviceId), Builders<SyncDevice>.Update
+                .Set(x => x.IsSyncReset, false));
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId, deviceId, "pending"));
+        }
+
+        public async Task<SyncDevice?> GetDeviceAsync(string userId, string deviceId)
+        {
+            return await repositories.SyncDevices.Collection.Find(DeviceFilter(userId, deviceId)).FirstOrDefaultAsync();
+        }
+
+        public async IAsyncEnumerable<SyncDevice> ListDevicesAsync(string userId)
+        {
+            using var cursor = await repositories.SyncDevices.Collection.FindAsync(UserFilter(userId));
+            while (await cursor.MoveNextAsync())
             {
-                if (IsSyncReset(id)) return;
-                lock (id)
+                foreach (var device in cursor.Current)
                 {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = GetUnsyncedIds(id);
-                    File.WriteAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds));
+                    yield return device;
                 }
             }
         }
 
-        public void RegisterDevice()
+        public async Task ResetDevicesAsync(string userId)
         {
-            lock (device.UserId)
+            await repositories.SyncDevices.Collection.DeleteManyAsync(UserFilter(userId));
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId));
+        }
+
+        public async Task UpdateLastAccessTimeAsync(string userId, string deviceId)
+        {
+            await repositories.SyncDevices.Collection.UpdateOneAsync(DeviceFilter(userId, deviceId), Builders<SyncDevice>.Update
+                .Set(x => x.LastAccessTime, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()));
+        }
+
+        public async Task AddIdsToOtherDevicesAsync(string userId, string deviceId, IEnumerable<ItemKey> ids)
+        {
+            await UpdateLastAccessTimeAsync(userId, deviceId);
+            await foreach (var device in ListDevicesAsync(userId))
             {
-                if (Directory.Exists(device.UserDeviceDirectoryPath))
-                    Directory.Delete(device.UserDeviceDirectoryPath, true);
-                Directory.CreateDirectory(device.UserDeviceDirectoryPath);
-                File.Create(device.ResetSyncFilePath).Close();
-                device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+                if (device.DeviceId == deviceId || device.IsSyncReset) continue;
+                await AppendIdsAsync(userId, device.DeviceId, "unsynced", ids);
             }
         }
 
-        public void UnregisterDevice()
+        public async Task AddIdsToAllDevicesAsync(string userId, IEnumerable<ItemKey> ids)
         {
-            lock (device.UserId)
+            await foreach (var device in ListDevicesAsync(userId))
             {
-                if (!Path.Exists(device.UserDeviceDirectoryPath)) return;
-                Directory.Delete(device.UserDeviceDirectoryPath, true);
+                if (device.IsSyncReset) continue;
+                await AppendIdsAsync(userId, device.DeviceId, "unsynced", ids);
             }
         }
+
+        public async Task<SyncDevice> RegisterDeviceAsync(string userId, string deviceId)
+        {
+            var newDevice = new SyncDevice
+            {
+                UserId = userId,
+                DeviceId = deviceId,
+                LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                IsSyncReset = true
+            };
+            await repositories.SyncDevices.Collection.InsertOneAsync(newDevice);
+            return newDevice;
+        }
+
+        public async Task UnregisterDeviceAsync(string userId, string deviceId)
+        {
+            await repositories.SyncDevices.Collection.DeleteOneAsync(DeviceFilter(userId, deviceId));
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId, deviceId));
+        }
     }
 }

Notesnook.API/Services/UserService.cs

@@ -23,47 +23,39 @@ using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
 using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
 using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Common.Models;
 using Streetwriters.Data.Interfaces;
 
 namespace Notesnook.API.Services
 {
-    public class UserService : IUserService
+    public class UserService(IHttpContextAccessor accessor,
+        ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor,
+        IUnitOfWork unitOfWork, IS3Service s3Service, SyncDeviceService syncDeviceService, WampServiceAccessor serviceAccessor, ILogger<UserService> logger) : IUserService
     {
         private static readonly System.Security.Cryptography.RandomNumberGenerator Rng = System.Security.Cryptography.RandomNumberGenerator.Create();
-        private readonly HttpClient httpClient;
-        private IHttpContextAccessor HttpContextAccessor { get; }
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private IS3Service S3Service { get; set; }
-        private readonly IUnitOfWork unit;
-
-        public UserService(IHttpContextAccessor accessor,
-        ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor,
-        IUnitOfWork unitOfWork, IS3Service s3Service)
-        {
-            httpClient = new HttpClient();
-
-            Repositories = syncItemsRepositoryAccessor;
-            HttpContextAccessor = accessor;
-            unit = unitOfWork;
-            S3Service = s3Service;
-        }
+        private readonly HttpClient httpClient = new();
+        private IHttpContextAccessor HttpContextAccessor { get; } = accessor;
+        private ISyncItemsRepositoryAccessor Repositories { get; } = syncItemsRepositoryAccessor;
+        private IS3Service S3Service { get; set; } = s3Service;
+        private readonly IUnitOfWork unit = unitOfWork;
 
         public async Task CreateUserAsync()
         {
             SignupResponse response = await httpClient.ForwardAsync<SignupResponse>(this.HttpContextAccessor, $"{Servers.IdentityServer}/signup", HttpMethod.Post);
-            if (!response.Success || (response.Errors != null && response.Errors.Length > 0))
+            if (!response.Success || (response.Errors != null && response.Errors.Length > 0) || response.UserId == null)
             {
-                await Slogger<UserService>.Error(nameof(CreateUserAsync), "Couldn't sign up.", JsonSerializer.Serialize(response));
+                logger.LogError("Failed to sign up user: {Response}", JsonSerializer.Serialize(response));
                 if (response.Errors != null && response.Errors.Length > 0)
                     throw new Exception(string.Join(" ", response.Errors));
                 else throw new Exception("Could not create a new account. Error code: " + response.StatusCode);
@@ -91,14 +83,12 @@ namespace Notesnook.API.Services
                 });
             }
 
-            await Slogger<UserService>.Info(nameof(CreateUserAsync), "New user created.", JsonSerializer.Serialize(response));
+            logger.LogInformation("New user created: {Response}", JsonSerializer.Serialize(response));
         }
 
         public async Task<UserResponse> GetUserAsync(string userId)
         {
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
-
-            var user = await userService.GetUserAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User not found.");
+            var user = await serviceAccessor.UserAccountService.GetUserAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User not found.");
 
             Subscription? subscription = null;
             if (Constants.IS_SELF_HOSTED)
@@ -117,17 +107,20 @@ namespace Notesnook.API.Services
             }
             else
             {
-                var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
-                subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
+                subscription = await serviceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
             }
 
             var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == user.UserId) ?? throw new Exception("User settings not found.");
 
             // reset user's attachment limit every month
-            if (userSettings.StorageLimit == null || DateTimeOffset.UtcNow.Month > DateTimeOffset.FromUnixTimeMilliseconds(userSettings.StorageLimit.UpdatedAt).Month)
+            var limit = StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+            if (userSettings.StorageLimit == null || limit.UpdatedAt != userSettings.StorageLimit?.UpdatedAt)
             {
-                userSettings.StorageLimit ??= new Limit { UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), Value = 0 };
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == user.UserId);
+                userSettings.StorageLimit = limit;
+                await Repositories.UsersSettings.Collection.UpdateOneAsync(
+                    Builders<UserSettings>.Filter.Eq(u => u.UserId, user.UserId),
+                    Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
+                );
             }
 
             return new UserResponse
@@ -189,8 +182,7 @@ namespace Notesnook.API.Services
 
         public async Task DeleteUserAsync(string userId)
         {
-            new SyncDeviceService(new SyncDevice(userId, userId)).ResetDevices();
-
+            logger.LogInformation("Deleting user {UserId}", userId);
             var cc = new CancellationTokenSource();
 
             Repositories.Notes.DeleteByUserId(userId);
@@ -210,9 +202,11 @@ namespace Notesnook.API.Services
             Repositories.InboxApiKey.DeleteMany((t) => t.UserId == userId);
 
             var result = await unit.Commit();
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "User data deleted", userId, result.ToString());
+            logger.LogInformation("User data deleted for user {UserId}: {Result}", userId, result);
             if (!result) throw new Exception("Could not delete user data.");
 
+            await syncDeviceService.ResetDevicesAsync(userId);
+
             if (!Constants.IS_SELF_HOSTED)
             {
                 await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.DeleteSubscriptionTopic, new DeleteSubscriptionMessage
@@ -225,18 +219,17 @@ namespace Notesnook.API.Services
             await S3Service.DeleteDirectoryAsync(userId);
         }
 
-        public async Task DeleteUserAsync(string userId, string jti, string password)
+        public async Task DeleteUserAsync(string userId, string? jti, string password)
         {
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "Deleting user account", userId);
+            logger.LogInformation("Deleting user account: {UserId}", userId);
 
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
-            await userService.DeleteUserAsync(Clients.Notesnook.Id, userId, password);
+            await serviceAccessor.UserAccountService.DeleteUserAsync(Clients.Notesnook.Id, userId, password);
 
             await DeleteUserAsync(userId);
 
             await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
             {
-                SendToAll = false,
+                SendToAll = jti == null,
                 OriginTokenId = jti,
                 UserId = userId,
                 Message = new Message
@@ -250,7 +243,6 @@ namespace Notesnook.API.Services
 
         public async Task<bool> ResetUserAsync(string userId, bool removeAttachments)
         {
-            new SyncDeviceService(new SyncDevice(userId, userId)).ResetDevices();
 
             var cc = new CancellationTokenSource();
 
@@ -270,6 +262,8 @@ namespace Notesnook.API.Services
             Repositories.InboxApiKey.DeleteMany((t) => t.UserId == userId);
             if (!await unit.Commit()) return false;
 
+            await syncDeviceService.ResetDevicesAsync(userId);
+
             var userSettings = await Repositories.UsersSettings.FindOneAsync((s) => s.UserId == userId);
 
             userSettings.AttachmentsKey = null;

Notesnook.API/Startup.cs

@@ -24,6 +24,7 @@ using System.Security.Claims;
 using System.Text;
 using System.Text.Encodings.Web;
 using System.Threading.Tasks;
+using Amazon.Runtime;
 using IdentityModel.AspNetCore.OAuth2Introspection;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
@@ -39,6 +40,7 @@ using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -119,8 +121,8 @@ namespace Notesnook.API
                     policy.RequireAuthenticatedUser();
                 });
 
-                options.DefaultPolicy = options.GetPolicy("Notesnook");
-            }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>(); ;
+                options.DefaultPolicy = options.GetPolicy("Notesnook") ?? throw new Exception("Notesnook policy not found");
+            }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>();
 
             services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
             .AddOAuth2Introspection("introspection", options =>
@@ -138,13 +140,13 @@ namespace Notesnook.API
 
                 options.Events.OnTokenValidated = (context) =>
                 {
-                    if (long.TryParse(context.Principal.FindFirst("exp")?.Value, out long expiryTime))
+                    if (long.TryParse(context.Principal?.FindFirst("exp")?.Value, out long expiryTime))
                     {
                         context.Properties.ExpiresUtc = DateTimeOffset.FromUnixTimeSeconds(expiryTime);
                     }
                     context.Properties.AllowRefresh = true;
                     context.Properties.IsPersistent = true;
-                    context.HttpContext.User = context.Principal;
+                    context.HttpContext.User = context.Principal ?? throw new Exception("No principal found in token.");
                     return Task.CompletedTask;
                 };
                 options.CacheKeyGenerator = (options, token) => (token + ":" + "reference_token").Sha256();
@@ -167,14 +169,19 @@ namespace Notesnook.API
             if (!BsonClassMap.IsClassMapRegistered(typeof(CallToAction)))
                 BsonClassMap.RegisterClassMap<CallToAction>();
 
+            if (!BsonClassMap.IsClassMapRegistered(typeof(SyncDevice)))
+                BsonClassMap.RegisterClassMap<SyncDevice>();
+
             services.AddScoped<IDbContext, MongoDbContext>();
             services.AddScoped<IUnitOfWork, UnitOfWork>();
 
             services.AddRepository<UserSettings>("user_settings", "notesnook")
                     .AddRepository<Monograph>("monographs", "notesnook")
                     .AddRepository<Announcement>("announcements", "notesnook")
+                    .AddRepository<DeviceIdsChunk>(Collections.DeviceIdsChunksKey, "notesnook")
+                    .AddRepository<SyncDevice>(Collections.SyncDevicesKey, "notesnook")
                     .AddRepository<InboxApiKey>(Collections.InboxApiKeysKey, "notesnook")
-                    .AddRepository<InboxSyncItem>(Collections.InboxItems, "notesnook");
+                    .AddRepository<InboxSyncItem>(Collections.InboxItemsKey, "notesnook");
 
             services.AddMongoCollection(Collections.SettingsKey)
                     .AddMongoCollection(Collections.AttachmentsKey)
@@ -188,17 +195,21 @@ namespace Notesnook.API
                     .AddMongoCollection(Collections.TagsKey)
                     .AddMongoCollection(Collections.ColorsKey)
                     .AddMongoCollection(Collections.VaultsKey)
-                    .AddMongoCollection(Collections.InboxItems)
+                    .AddMongoCollection(Collections.InboxItemsKey)
                     .AddMongoCollection(Collections.InboxApiKeysKey);
 
             services.AddScoped<ISyncItemsRepositoryAccessor, SyncItemsRepositoryAccessor>();
+            services.AddScoped<SyncDeviceService>();
             services.AddScoped<IUserService, UserService>();
             services.AddScoped<IS3Service, S3Service>();
             services.AddScoped<IURLAnalyzer, URLAnalyzer>();
 
+            services.AddWampServiceAccessor(Servers.NotesnookAPI);
+
             services.AddControllers();
 
-            services.AddHealthChecks(); // .AddMongoDb(dbSettings.ConnectionString, dbSettings.DatabaseName, "database-check");
+            services.AddHealthChecks();
+
             services.AddSignalR((hub) =>
             {
                 hub.MaximumReceiveMessageSize = 100 * 1024 * 1024;
@@ -251,13 +262,7 @@ namespace Notesnook.API
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
         public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         {
-            if (!env.IsDevelopment())
-            {
-                app.UseForwardedHeaders(new ForwardedHeadersOptions
-                {
-                    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
-                });
-            }
+            app.UseForwardedHeadersWithKnownProxies(env);
 
             app.UseOpenTelemetryPrometheusScrapingEndpoint((context) => context.Request.Path == "/metrics" && context.Connection.LocalPort == 5067);
             app.UseResponseCompression();
@@ -289,11 +294,6 @@ namespace Notesnook.API
             {
                 endpoints.MapControllers();
                 endpoints.MapHealthChecks("/health");
-                endpoints.MapHub<SyncHub>("/hubs/sync", options =>
-                {
-                    options.CloseOnAuthenticationExpiration = false;
-                    options.Transports = HttpTransportType.WebSockets;
-                });
                 endpoints.MapHub<SyncV2Hub>("/hubs/sync/v2", options =>
                 {
                     options.CloseOnAuthenticationExpiration = false;
@@ -307,7 +307,7 @@ namespace Notesnook.API
     {
         public static IServiceCollection AddMongoCollection(this IServiceCollection services, string collectionName, string database = "notesnook")
         {
-            services.AddKeyedSingleton(collectionName, (provider, key) => MongoDbContext.GetMongoCollection<SyncItem>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
+            services.AddKeyedSingleton(collectionName, (provider, key) => MongoDbContext.GetMongoCollection<SyncItem>(provider.GetRequiredService<MongoDB.Driver.IMongoClient>(), database, collectionName));
             return services;
         }
     }

Notesnook.API/appsettings.Development.json

@@ -1,7 +1,7 @@
 {
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Debug",
       "Microsoft": "Warning",
       "Microsoft.Hosting.Lifetime": "Information",
       "Microsoft.AspNetCore.SignalR": "Trace",

Notesnook.Inbox.API/bun.lock

@@ -5,6 +5,7 @@
       "name": "notesnook-inbox-api",
       "dependencies": {
         "express": "^5.1.0",
+        "express-rate-limit": "^8.1.0",
         "libsodium-wrappers-sumo": "^0.7.15",
         "zod": "^4.1.9",
       },
@@ -83,6 +84,8 @@
 
     "express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="],
 
+    "express-rate-limit": ["express-rate-limit@8.1.0", "", { "dependencies": { "ip-address": "10.0.1" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-4nLnATuKupnmwqiJc27b4dCFmB/T60ExgmtDD7waf4LdrbJ8CPZzZRHYErDYNhoz+ql8fUdYwM/opf90PoPAQA=="],
+
     "finalhandler": ["finalhandler@2.1.0", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q=="],
 
     "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
@@ -107,6 +110,8 @@
 
     "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
 
+    "ip-address": ["ip-address@10.0.1", "", {}, "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA=="],
+
     "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
 
     "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],

Notesnook.Inbox.API/package.json

@@ -21,6 +21,7 @@
   },
   "dependencies": {
     "express": "^5.1.0",
+    "express-rate-limit": "^8.1.0",
     "libsodium-wrappers-sumo": "^0.7.15",
     "zod": "^4.1.9"
   },

Notesnook.Inbox.API/src/index.ts

@@ -1,6 +1,7 @@
 import express from "express";
 import _sodium, { base64_variants } from "libsodium-wrappers-sumo";
 import { z } from "zod";
+import { rateLimit } from "express-rate-limit";
 
 const NOTESNOOK_API_SERVER_URL = process.env.NOTESNOOK_API_SERVER_URL;
 if (!NOTESNOOK_API_SERVER_URL) {
@@ -30,16 +31,26 @@ const RawInboxItemSchema = z.object({
 
 interface EncryptedInboxItem {
   v: 1;
-  key: Omit<EncryptedInboxItem, "key" | "iv" | "v">;
+  key: Omit<EncryptedInboxItem, "key" | "iv" | "v" | "salt">;
   iv: string;
   alg: string;
   cipher: string;
   length: number;
+  salt: string;
 }
 
 function encrypt(rawData: string, publicKey: string): EncryptedInboxItem {
   try {
     const password = sodium.crypto_aead_xchacha20poly1305_ietf_keygen();
+    const saltBytes = sodium.randombytes_buf(sodium.crypto_pwhash_SALTBYTES);
+    const key = sodium.crypto_pwhash(
+      sodium.crypto_aead_xchacha20poly1305_ietf_KEYBYTES,
+      password,
+      saltBytes,
+      3, // operations limit
+      1024 * 1024 * 8, // memory limit (8MB)
+      sodium.crypto_pwhash_ALG_ARGON2I13
+    );
     const nonce = sodium.randombytes_buf(
       sodium.crypto_aead_xchacha20poly1305_ietf_NPUBBYTES
     );
@@ -49,19 +60,19 @@ function encrypt(rawData: string, publicKey: string): EncryptedInboxItem {
       null,
       null,
       nonce,
-      password
+      key
     );
     const inboxPublicKey = sodium.from_base64(
       publicKey,
       base64_variants.URLSAFE_NO_PADDING
     );
-    const encryptedPassword = sodium.crypto_box_seal(password, inboxPublicKey);
+    const encryptedKey = sodium.crypto_box_seal(key, inboxPublicKey);
 
     return {
       v: 1,
       key: {
         cipher: sodium.to_base64(
-          encryptedPassword,
+          encryptedKey,
           base64_variants.URLSAFE_NO_PADDING
         ),
         alg: `xsal-x25519-${base64_variants.URLSAFE_NO_PADDING}`,
@@ -71,6 +82,7 @@ function encrypt(rawData: string, publicKey: string): EncryptedInboxItem {
       alg: `xcha-argon2i13-${base64_variants.URLSAFE_NO_PADDING}`,
       cipher: sodium.to_base64(cipher, base64_variants.URLSAFE_NO_PADDING),
       length: data.length,
+      salt: sodium.to_base64(saltBytes, base64_variants.URLSAFE_NO_PADDING),
     };
   } catch (error) {
     throw new Error(`encryption failed: ${error}`);
@@ -79,7 +91,7 @@ function encrypt(rawData: string, publicKey: string): EncryptedInboxItem {
 
 async function getInboxPublicEncryptionKey(apiKey: string) {
   const response = await fetch(
-    `${NOTESNOOK_API_SERVER_URL}inbox/public-encryption-key`,
+    `${NOTESNOOK_API_SERVER_URL}/inbox/public-encryption-key`,
     {
       headers: {
         Authorization: apiKey,
@@ -100,7 +112,7 @@ async function postEncryptedInboxItem(
   apiKey: string,
   item: EncryptedInboxItem
 ) {
-  const response = await fetch(`${NOTESNOOK_API_SERVER_URL}inbox/items`, {
+  const response = await fetch(`${NOTESNOOK_API_SERVER_URL}/inbox/items`, {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
@@ -115,17 +127,26 @@ async function postEncryptedInboxItem(
 
 const app = express();
 app.use(express.json({ limit: "10mb" }));
+app.use(
+  rateLimit({
+    windowMs: 1 * 60 * 1000, // 1 minute
+    limit: 60,
+  })
+);
 app.post("/inbox", async (req, res) => {
   try {
     const apiKey = req.headers["authorization"];
     if (!apiKey) {
       return res.status(401).json({ error: "unauthorized" });
     }
-    if (!req.body.item) {
-      return res.status(400).json({ error: "item is required" });
-    }
 
-    const validationResult = RawInboxItemSchema.safeParse(req.body.item);
+    const inboxPublicKey = await getInboxPublicEncryptionKey(apiKey);
+    if (!inboxPublicKey) {
+      return res.status(403).json({ error: "inbox public key not found" });
+    }
+    console.log("[info] fetched inbox public key");
+
+    const validationResult = RawInboxItemSchema.safeParse(req.body);
     if (!validationResult.success) {
       return res.status(400).json({
         error: "invalid item",
@@ -133,17 +154,16 @@ app.post("/inbox", async (req, res) => {
       });
     }
 
-    const inboxPublicKey = await getInboxPublicEncryptionKey(apiKey);
-    if (!inboxPublicKey) {
-      return res.status(403).json({ error: "inbox public key not found" });
-    }
-    console.log("[info] fetched inbox public key:", inboxPublicKey);
+    const encryptedItem = encrypt(
+      JSON.stringify(validationResult.data),
+      inboxPublicKey
+    );
+    console.log("[info] encrypted item");
 
-    const item = validationResult.data;
-    const encryptedItem = encrypt(JSON.stringify(item), inboxPublicKey);
-    console.log("[info] encrypted item:", encryptedItem);
     await postEncryptedInboxItem(apiKey, encryptedItem);
-    return res.status(200).json({ message: "inbox item posted" });
+    console.log("[info] posted encrypted inbox item successfully");
+
+    return res.status(200).json({ success: true });
   } catch (error) {
     if (error instanceof Error) {
       console.log("[error]", error.message);

Streetwriters.Common/Accessors/WampServiceAccessor.cs

@@ -0,0 +1,48 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Streetwriters.Common.Interfaces;
+
+namespace Streetwriters.Common.Accessors
+{
+    public class WampServiceAccessor(Server server) : IHostedService
+    {
+        public IUserAccountService UserAccountService { get; set; }
+        public IUserSubscriptionService? UserSubscriptionService { get; set; }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            await InitAsync();
+        }
+
+        private async Task InitAsync()
+        {
+            this.UserAccountService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(InitAsync);
+            if (!Constants.IS_SELF_HOSTED && server != Servers.SubscriptionServer)
+            {
+                this.UserSubscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(InitAsync);
+            }
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+    }
+}

Streetwriters.Common/Attributes/JsonInterfaceConverterAttribute.cs

@@ -1,33 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Text.Json.Serialization;
-
-namespace Streetwriters.Common.Attributes
-{
-    [AttributeUsage(AttributeTargets.Interface, AllowMultiple = false)]
-    public class JsonInterfaceConverterAttribute : JsonConverterAttribute
-    {
-        public JsonInterfaceConverterAttribute(Type converterType)
-            : base(converterType)
-        {
-        }
-    }
-}

Streetwriters.Common/Clients.cs

@@ -33,7 +33,7 @@ namespace Streetwriters.Common
         {
             Id = "notesnook",
             Name = "Notesnook",
-            SenderEmail = Constants.NOTESNOOK_SENDER_EMAIL,
+            SenderEmail = Constants.NOTESNOOK_SENDER_EMAIL ?? "noreply@notesnook.com",
             SenderName = "Notesnook",
             Type = ApplicationType.NOTESNOOK,
             AppId = ApplicationType.NOTESNOOK,
@@ -58,14 +58,15 @@ namespace Streetwriters.Common
             { "notesnook", Notesnook }
         };
 
-        public static Client FindClientById(string id)
+        public static Client? FindClientById(string? id)
         {
-            if (!IsValidClient(id)) return null;
+            if (string.IsNullOrEmpty(id) || !IsValidClient(id)) return null;
             return ClientsMap[id];
         }
 
-        public static Client FindClientByAppId(ApplicationType appId)
+        public static Client? FindClientByAppId(ApplicationType? appId)
         {
+            if (appId is null) return null;
             switch (appId)
             {
                 case ApplicationType.NOTESNOOK:

Streetwriters.Common/Constants.cs

@@ -24,59 +24,73 @@ namespace Streetwriters.Common
     public class Constants
     {
         public static int COMPATIBILITY_VERSION = 1;
-        public static bool IS_SELF_HOSTED => Environment.GetEnvironmentVariable("SELF_HOSTED") == "1";
-        public static bool DISABLE_SIGNUPS => Environment.GetEnvironmentVariable("DISABLE_SIGNUPS") == "true";
-        public static string INSTANCE_NAME => Environment.GetEnvironmentVariable("INSTANCE_NAME") ?? "default";
+        public static bool IS_SELF_HOSTED => ReadSecret("SELF_HOSTED") == "1";
+        public static bool DISABLE_SIGNUPS => ReadSecret("DISABLE_SIGNUPS") == "true";
+        public static string INSTANCE_NAME => ReadSecret("INSTANCE_NAME") ?? "default";
 
         // S3 related
-        public static string S3_ACCESS_KEY => Environment.GetEnvironmentVariable("S3_ACCESS_KEY");
-        public static string S3_ACCESS_KEY_ID => Environment.GetEnvironmentVariable("S3_ACCESS_KEY_ID");
-        public static string S3_SERVICE_URL => Environment.GetEnvironmentVariable("S3_SERVICE_URL");
-        public static string S3_REGION => Environment.GetEnvironmentVariable("S3_REGION");
-        public static string S3_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_BUCKET_NAME");
-        public static string S3_INTERNAL_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_INTERNAL_BUCKET_NAME");
-        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
+        public static string S3_ACCESS_KEY => ReadSecret("S3_ACCESS_KEY") ?? throw new InvalidOperationException("S3_ACCESS_KEY is required");
+        public static string S3_ACCESS_KEY_ID => ReadSecret("S3_ACCESS_KEY_ID") ?? throw new InvalidOperationException("S3_ACCESS_KEY_ID is required");
+        public static string S3_SERVICE_URL => ReadSecret("S3_SERVICE_URL") ?? throw new InvalidOperationException("S3_SERVICE_URL is required");
+        public static string S3_REGION => ReadSecret("S3_REGION") ?? throw new InvalidOperationException("S3_REGION is required");
+        public static string S3_BUCKET_NAME => ReadSecret("S3_BUCKET_NAME") ?? throw new InvalidOperationException("S3_BUCKET_NAME is required");
+        public static string? S3_INTERNAL_BUCKET_NAME => ReadSecret("S3_INTERNAL_BUCKET_NAME");
+        public static string? S3_INTERNAL_SERVICE_URL => ReadSecret("S3_INTERNAL_SERVICE_URL");
 
         // SMTP settings
-        public static string SMTP_USERNAME => Environment.GetEnvironmentVariable("SMTP_USERNAME");
-        public static string SMTP_PASSWORD => Environment.GetEnvironmentVariable("SMTP_PASSWORD");
-        public static string SMTP_HOST => Environment.GetEnvironmentVariable("SMTP_HOST");
-        public static string SMTP_PORT => Environment.GetEnvironmentVariable("SMTP_PORT");
-        public static string SMTP_REPLYTO_EMAIL => Environment.GetEnvironmentVariable("SMTP_REPLYTO_EMAIL");
-        public static string NOTESNOOK_SENDER_EMAIL => Environment.GetEnvironmentVariable("NOTESNOOK_SENDER_EMAIL") ?? Environment.GetEnvironmentVariable("SMTP_USERNAME");
+        public static string? SMTP_USERNAME => ReadSecret("SMTP_USERNAME");
+        public static string? SMTP_PASSWORD => ReadSecret("SMTP_PASSWORD");
+        public static string? SMTP_HOST => ReadSecret("SMTP_HOST");
+        public static string? SMTP_PORT => ReadSecret("SMTP_PORT");
+        public static string? SMTP_REPLYTO_EMAIL => ReadSecret("SMTP_REPLYTO_EMAIL");
+        public static string? NOTESNOOK_SENDER_EMAIL => ReadSecret("NOTESNOOK_SENDER_EMAIL") ?? ReadSecret("SMTP_USERNAME");
 
-        public static string NOTESNOOK_APP_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_APP_HOST");
-        public static string NOTESNOOK_API_SECRET => Environment.GetEnvironmentVariable("NOTESNOOK_API_SECRET");
+        public static string? NOTESNOOK_APP_HOST => ReadSecret("NOTESNOOK_APP_HOST");
+        public static string NOTESNOOK_API_SECRET => ReadSecret("NOTESNOOK_API_SECRET") ?? throw new InvalidOperationException("NOTESNOOK_API_SECRET is required");
 
         // MessageBird is used for SMS sending
-        public static string TWILIO_ACCOUNT_SID => Environment.GetEnvironmentVariable("TWILIO_ACCOUNT_SID");
-        public static string TWILIO_AUTH_TOKEN => Environment.GetEnvironmentVariable("TWILIO_AUTH_TOKEN");
-        public static string TWILIO_SERVICE_SID => Environment.GetEnvironmentVariable("TWILIO_SERVICE_SID");
+        public static string? TWILIO_ACCOUNT_SID => ReadSecret("TWILIO_ACCOUNT_SID");
+        public static string? TWILIO_AUTH_TOKEN => ReadSecret("TWILIO_AUTH_TOKEN");
+        public static string? TWILIO_SERVICE_SID => ReadSecret("TWILIO_SERVICE_SID");
         // Server discovery
-        public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT") ?? "80");
-        public static string NOTESNOOK_SERVER_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_HOST");
-        public static string NOTESNOOK_CERT_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_PATH");
-        public static string NOTESNOOK_CERT_KEY_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_KEY_PATH");
+        public static int NOTESNOOK_SERVER_PORT => int.Parse(ReadSecret("NOTESNOOK_SERVER_PORT") ?? "80");
+        public static string? NOTESNOOK_SERVER_HOST => ReadSecret("NOTESNOOK_SERVER_HOST");
+        public static string? NOTESNOOK_CERT_PATH => ReadSecret("NOTESNOOK_CERT_PATH");
+        public static string? NOTESNOOK_CERT_KEY_PATH => ReadSecret("NOTESNOOK_CERT_KEY_PATH");
+        public static string[] KNOWN_PROXIES => (ReadSecret("KNOWN_PROXIES") ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
 
-        public static int IDENTITY_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("IDENTITY_SERVER_PORT") ?? "80");
-        public static string IDENTITY_SERVER_HOST => Environment.GetEnvironmentVariable("IDENTITY_SERVER_HOST");
-        public static Uri IDENTITY_SERVER_URL => new(Environment.GetEnvironmentVariable("IDENTITY_SERVER_URL"));
-        public static string IDENTITY_CERT_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_PATH");
-        public static string IDENTITY_CERT_KEY_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_KEY_PATH");
+        public static int IDENTITY_SERVER_PORT => int.Parse(ReadSecret("IDENTITY_SERVER_PORT") ?? "80");
+        public static string? IDENTITY_SERVER_HOST => ReadSecret("IDENTITY_SERVER_HOST");
+        public static Uri? IDENTITY_SERVER_URL => ReadSecret("IDENTITY_SERVER_URL") is string url ? new Uri(url) : null;
+        public static string? IDENTITY_CERT_PATH => ReadSecret("IDENTITY_CERT_PATH");
+        public static string? IDENTITY_CERT_KEY_PATH => ReadSecret("IDENTITY_CERT_KEY_PATH");
 
-        public static int SSE_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SSE_SERVER_PORT") ?? "80");
-        public static string SSE_SERVER_HOST => Environment.GetEnvironmentVariable("SSE_SERVER_HOST");
-        public static string SSE_CERT_PATH => Environment.GetEnvironmentVariable("SSE_CERT_PATH");
-        public static string SSE_CERT_KEY_PATH => Environment.GetEnvironmentVariable("SSE_CERT_KEY_PATH");
+        public static int SSE_SERVER_PORT => int.Parse(ReadSecret("SSE_SERVER_PORT") ?? "80");
+        public static string? SSE_SERVER_HOST => ReadSecret("SSE_SERVER_HOST");
+        public static string? SSE_CERT_PATH => ReadSecret("SSE_CERT_PATH");
+        public static string? SSE_CERT_KEY_PATH => ReadSecret("SSE_CERT_KEY_PATH");
 
         // internal
-        public static string WEBRISK_API_URI => Environment.GetEnvironmentVariable("WEBRISK_API_URI");
-        public static string MONGODB_CONNECTION_STRING => Environment.GetEnvironmentVariable("MONGODB_CONNECTION_STRING");
-        public static string MONGODB_DATABASE_NAME => Environment.GetEnvironmentVariable("MONGODB_DATABASE_NAME");
-        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_PORT") ?? "80");
-        public static string SUBSCRIPTIONS_SERVER_HOST => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_HOST");
-        public static string SUBSCRIPTIONS_CERT_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_PATH");
-        public static string SUBSCRIPTIONS_CERT_KEY_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_KEY_PATH");
-        public static string[] NOTESNOOK_CORS_ORIGINS => Environment.GetEnvironmentVariable("NOTESNOOK_CORS")?.Split(",") ?? new string[] { };
+        public static string? WEBRISK_API_URI => ReadSecret("WEBRISK_API_URI");
+        public static string MONGODB_CONNECTION_STRING => ReadSecret("MONGODB_CONNECTION_STRING") ?? throw new ArgumentNullException("MONGODB_CONNECTION_STRING environment variable is not set");
+        public static string MONGODB_DATABASE_NAME => ReadSecret("MONGODB_DATABASE_NAME") ?? throw new ArgumentNullException("MONGODB_DATABASE_NAME environment variable is not set");
+        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(ReadSecret("SUBSCRIPTIONS_SERVER_PORT") ?? "80");
+        public static string? SUBSCRIPTIONS_SERVER_HOST => ReadSecret("SUBSCRIPTIONS_SERVER_HOST");
+        public static string? SUBSCRIPTIONS_CERT_PATH => ReadSecret("SUBSCRIPTIONS_CERT_PATH");
+        public static string? SUBSCRIPTIONS_CERT_KEY_PATH => ReadSecret("SUBSCRIPTIONS_CERT_KEY_PATH");
+        public static string[] NOTESNOOK_CORS_ORIGINS => ReadSecret("NOTESNOOK_CORS")?.Split(",") ?? [];
+
+        public static string? ReadSecret(string name)
+        {
+            var value = Environment.GetEnvironmentVariable(name);
+            if (!string.IsNullOrEmpty(value)) return value;
+            var file = Environment.GetEnvironmentVariable(name + "_FILE");
+            if (!string.IsNullOrEmpty(file) && System.IO.File.Exists(file))
+            {
+                return System.IO.File.ReadAllText(file);
+            }
+            return null;
+        }
     }
-}
+}
+

Streetwriters.Common/Converters/InterfaceConverter.cs

@@ -1,54 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Streetwriters.Common.Converters
-{
-    /// <summary>
-    /// Converts simple interface into an object (assumes that there is only one class of TInterface)
-    /// </summary>
-    /// <typeparam name="TInterface">Interface type</typeparam>
-    /// <typeparam name="TClass">Class type</typeparam>
-    public class InterfaceConverter<TInterface, TClass> : JsonConverter<TInterface> where TClass : TInterface
-    {
-        public override TInterface Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
-        {
-            return JsonSerializer.Deserialize<TClass>(ref reader, options);
-        }
-
-        public override void Write(Utf8JsonWriter writer, TInterface value, JsonSerializerOptions options)
-        {
-            switch (value)
-            {
-                case null:
-                    JsonSerializer.Serialize(writer, null, options);
-                    break;
-                default:
-                    {
-                        var type = value.GetType();
-                        JsonSerializer.Serialize(writer, value, type, options);
-                        break;
-                    }
-            }
-        }
-    }
-}

Streetwriters.Common/Extensions/AppBuilderExtensions.cs

@@ -19,10 +19,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Text.Json;
 using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
 using WampSharp.AspNetCore.WebSockets.Server;
 using WampSharp.Binding;
 using WampSharp.V2;
@@ -42,7 +46,7 @@ namespace Streetwriters.Common.Extensions
                     var data = new Dictionary<string, object>
                     {
                         { "version", Constants.COMPATIBILITY_VERSION },
-                        { "id", server.Id },
+                        { "id", server.Id ?? "unknown" },
                         { "instance", Constants.INSTANCE_NAME }
                     };
                     await context.Response.WriteAsync(JsonSerializer.Serialize(data));
@@ -51,9 +55,9 @@ namespace Streetwriters.Common.Extensions
             return app;
         }
 
-        public static IApplicationBuilder UseWamp<T>(this IApplicationBuilder app, WampServer<T> server, Action<IWampHostedRealm, WampServer<T>> action) where T : new()
+        public static IApplicationBuilder UseWamp(this IApplicationBuilder app, WampServer server, Action<IWampHostedRealm, WampServer> action)
         {
-            WampHost host = new WampHost();
+            WampHost host = new();
 
             app.Map(server.Endpoint, builder =>
             {
@@ -70,17 +74,40 @@ namespace Streetwriters.Common.Extensions
             return app;
         }
 
-        public static T GetService<T>(this IApplicationBuilder app)
+        public static T GetService<T>(this IApplicationBuilder app) where T : notnull
         {
             return app.ApplicationServices.GetRequiredService<T>();
         }
 
-        public static T GetScopedService<T>(this IApplicationBuilder app)
+        public static T GetScopedService<T>(this IApplicationBuilder app) where T : notnull
         {
-            using (var scope = app.ApplicationServices.CreateScope())
+            using var scope = app.ApplicationServices.CreateScope();
+            return scope.ServiceProvider.GetRequiredService<T>();
+        }
+
+        public static IApplicationBuilder UseForwardedHeadersWithKnownProxies(this IApplicationBuilder app, IWebHostEnvironment env, string forwardedForHeaderName = null)
+        {
+            if (!env.IsDevelopment())
             {
-                return scope.ServiceProvider.GetRequiredService<T>();
+                var forwardedHeadersOptions = new ForwardedHeadersOptions
+                {
+                    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+                };
+
+                if (!string.IsNullOrEmpty(forwardedForHeaderName))
+                {
+                    forwardedHeadersOptions.ForwardedForHeaderName = forwardedForHeaderName;
+                }
+
+                foreach (var proxy in Constants.KNOWN_PROXIES)
+                {
+                    forwardedHeadersOptions.KnownProxies.Add(IPAddress.Parse(proxy));
+                }
+
+                app.UseForwardedHeaders(forwardedHeadersOptions);
             }
+
+            return app;
         }
     }
 }

Streetwriters.Common/Extensions/HttpClientExtensions.cs

@@ -17,6 +17,7 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Json;
@@ -28,7 +29,7 @@ namespace Streetwriters.Common.Extensions
 {
     public static class HttpClientExtensions
     {
-        public static async Task<T> SendRequestAsync<T>(this HttpClient httpClient, string url, IHeaderDictionary headers, HttpMethod method, HttpContent content = null) where T : IResponse, new()
+        public static async Task<T> SendRequestAsync<T>(this HttpClient httpClient, string url, IHeaderDictionary? headers, HttpMethod method, HttpContent? content = null) where T : IResponse, new()
         {
             var request = new HttpRequestMessage(method, url);
 
@@ -51,22 +52,23 @@ namespace Streetwriters.Common.Extensions
             }
 
             var response = await httpClient.SendAsync(request);
-            if (response.Content.Headers.ContentLength > 0 && response.Content.Headers.ContentType.ToString().Contains("application/json"))
+            if (response.Content.Headers.ContentLength > 0 && response.Content.Headers.ContentType?.ToString()?.Contains("application/json") == true)
             {
                 var res = await response.Content.ReadFromJsonAsync<T>();
-                res.Success = response.IsSuccessStatusCode;
-                res.StatusCode = (int)response.StatusCode;
-                return res;
-            }
-            else
-            {
-                return new T { Success = response.IsSuccessStatusCode, StatusCode = (int)response.StatusCode, Content = response.Content };
+                if (res != null)
+                {
+                    res.Success = response.IsSuccessStatusCode;
+                    res.StatusCode = (int)response.StatusCode;
+                    return res;
+                }
             }
+
+            return new T { Success = response.IsSuccessStatusCode, StatusCode = (int)response.StatusCode, Content = response.Content };
         }
 
         public static Task<T> ForwardAsync<T>(this HttpClient httpClient, IHttpContextAccessor accessor, string url, HttpMethod method) where T : IResponse, new()
         {
-            var httpContext = accessor.HttpContext;
+            var httpContext = accessor.HttpContext ?? throw new InvalidOperationException("HttpContext is not available");
             var content = new StreamContent(httpContext.Request.BodyReader.AsStream());
             return httpClient.SendRequestAsync<T>(url, httpContext.Request.Headers, method, content);
         }

Streetwriters.Common/Extensions/ServiceCollectionServiceExtensions.cs

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using Microsoft.Extensions.DependencyInjection;
+using Streetwriters.Common.Accessors;
 using Streetwriters.Data.DbContexts;
 using Streetwriters.Data.Repositories;
 
@@ -25,9 +26,16 @@ namespace Streetwriters.Common.Extensions
 {
     public static class ServiceCollectionServiceExtensions
     {
+        public static IServiceCollection AddWampServiceAccessor(this IServiceCollection services, Server server)
+        {
+            services.AddSingleton<WampServiceAccessor>((provider) => new WampServiceAccessor(server));
+            services.AddHostedService(provider => provider.GetRequiredService<WampServiceAccessor>());
+            return services;
+        }
+
         public static IServiceCollection AddRepository<T>(this IServiceCollection services, string collectionName, string database) where T : class
         {
-            services.AddSingleton((provider) => MongoDbContext.GetMongoCollection<T>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
+            services.AddSingleton((provider) => MongoDbContext.GetMongoCollection<T>(provider.GetRequiredService<MongoDB.Driver.IMongoClient>(), database, collectionName));
             services.AddScoped<Repository<T>>();
             return services;
         }

Streetwriters.Common/Helpers/FeatureAuthorizationHelper.cs

@@ -0,0 +1,62 @@
+using System.IO;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using Streetwriters.Common.Enums;
+using Streetwriters.Common.Interfaces;
+using WebMarkupMin.Core;
+using WebMarkupMin.Core.Loggers;
+
+namespace Streetwriters.Common.Helpers
+{
+    public enum Features
+    {
+        SMS_2FA,
+        MONOGRAPH_ANALYTICS
+    }
+
+    public static class FeatureAuthorizationHelper
+    {
+        private static SubscriptionPlan? GetUserSubscriptionPlan(string clientId, ClaimsPrincipal user)
+        {
+            var claimKey = $"{clientId}:status";
+            var status = user.FindFirstValue(claimKey);
+            switch (status)
+            {
+                case "free":
+                    return SubscriptionPlan.FREE;
+                case "believer":
+                    return SubscriptionPlan.BELIEVER;
+                case "education":
+                    return SubscriptionPlan.EDUCATION;
+                case "essential":
+                    return SubscriptionPlan.ESSENTIAL;
+                case "pro":
+                    return SubscriptionPlan.PRO;
+                case "legacy_pro":
+                    return SubscriptionPlan.LEGACY_PRO;
+                default:
+                    return null;
+            }
+        }
+
+        public static bool IsFeatureAllowed(Features feature, string clientId, ClaimsPrincipal user)
+        {
+            if (Constants.IS_SELF_HOSTED)
+                return true;
+
+            var status = GetUserSubscriptionPlan(clientId, user);
+
+            switch (feature)
+            {
+                case Features.SMS_2FA:
+                case Features.MONOGRAPH_ANALYTICS:
+                    return status == SubscriptionPlan.LEGACY_PRO ||
+                            status == SubscriptionPlan.PRO ||
+                            status == SubscriptionPlan.EDUCATION ||
+                            status == SubscriptionPlan.BELIEVER;
+                default:
+                    return false;
+            }
+        }
+    }
+}

Streetwriters.Common/Helpers/WampHelper.cs

@@ -28,15 +28,28 @@ namespace Streetwriters.Common.Helpers
 {
     public class WampHelper
     {
-        public static async Task<IWampRealmProxy> OpenWampChannelAsync(string server, string realmName)
+        public static async Task<IWampChannel> OpenWampChannelAsync(string server, string realmName)
         {
             DefaultWampChannelFactory channelFactory = new();
 
             IWampChannel channel = channelFactory.CreateJsonChannel(server, realmName);
 
-            await channel.Open();
+            var isConnected = false;
+            while (!isConnected)
+            {
+                try
+                {
+                    await channel.Open();
+                    isConnected = true;
+                }
+                catch
+                {
+                    await Task.Delay(5000);
+                    continue;
+                }
+            }
 
-            return channel.RealmProxy;
+            return channel;
         }
 
         public static void PublishMessage<T>(IWampRealmProxy realm, string topicName, T message)

Streetwriters.Common/Interfaces/IClient.cs

@@ -33,6 +33,6 @@ namespace Streetwriters.Common.Interfaces
         string SenderName { get; set; }
         string EmailConfirmedRedirectURL { get; }
         string AccountRecoveryRedirectURL { get; }
-        Func<string, Task> OnEmailConfirmed { get; set; }
+        Func<string, Task>? OnEmailConfirmed { get; set; }
     }
 }

Streetwriters.Common/Interfaces/IDocument.cs

@@ -17,11 +17,13 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using MongoDB.Bson;
+
 namespace Streetwriters.Common.Interfaces
 {
     public interface IDocument
     {
-        string Id
+        ObjectId Id
         {
             get; set;
         }

Streetwriters.Common/Interfaces/IEmailSender.cs

@@ -12,8 +12,8 @@ namespace Streetwriters.Common.Interfaces
             string email,
             EmailTemplate template,
             IClient client,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
+            GnuPGContext? gpgContext = null,
+            Dictionary<string, byte[]>? attachments = null
         );
     }
 }

Streetwriters.Common/Interfaces/IOffer.cs

@@ -1,32 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Collections.Generic;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Models;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IOffer : IDocument
-    {
-        ApplicationType AppId { get; set; }
-        string PromoCode { get; set; }
-        PromoCode[] Codes { get; set; }
-    }
-}

Streetwriters.Common/Interfaces/IResponse.cs

@@ -25,6 +25,6 @@ namespace Streetwriters.Common.Interfaces
     {
         bool Success { get; set; }
         int StatusCode { get; set; }
-        HttpContent Content { get; set; }
+        HttpContent? Content { get; set; }
     }
 }

Streetwriters.Common/Interfaces/IUserSubscriptionService.cs

@@ -8,7 +8,7 @@ namespace Streetwriters.Common.Interfaces
     public interface IUserSubscriptionService
     {
         [WampProcedure("co.streetwriters.subscriptions.subscriptions.get_user_subscription")]
-        Task<Subscription> GetUserSubscriptionAsync(string clientId, string userId);
+        Task<Subscription?> GetUserSubscriptionAsync(string clientId, string userId);
         Subscription TransformUserSubscription(Subscription subscription);
     }
 }

Streetwriters.Common/Logger.cs

@@ -1,52 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.IO;
-using System.Threading.Tasks;
-
-namespace Streetwriters.Common
-{
-    public class Slogger<T>
-    {
-        public static Task Info(string scope, params string[] messages)
-        {
-            return Write(Format("info", scope, messages));
-        }
-
-        public static Task Error(string scope, params string[] messages)
-        {
-            return Write(Format("error", scope, messages));
-        }
-        private static string Format(string level, string scope, params string[] messages)
-        {
-            var date = DateTime.UtcNow.ToString("MM-dd-yyyy HH:mm:ss");
-            var messageText = string.Join(" ", messages);
-            return $"[{date}] | {level} | <{scope}> {messageText}";
-        }
-        private static Task Write(string line)
-        {
-            var logDirectory = Path.GetFullPath("./logs");
-            if (!Directory.Exists(logDirectory))
-                Directory.CreateDirectory(logDirectory);
-            var path = Path.Join(logDirectory, typeof(T).FullName + "-" + DateTime.UtcNow.ToString("MM-dd-yyyy") + ".log");
-            return File.AppendAllLinesAsync(path, new string[1] { line });
-        }
-    }
-}

Streetwriters.Common/Messages/CreateSubscriptionMessage.cs

@@ -28,7 +28,7 @@ namespace Streetwriters.Common.Messages
     public class CreateSubscriptionMessage
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("provider")]
         public SubscriptionProvider Provider { get; set; }
@@ -46,19 +46,19 @@ namespace Streetwriters.Common.Messages
         public long ExpiryTime { get; set; }
 
         [JsonPropertyName("orderId")]
-        public string OrderId { get; set; }
+        public string? OrderId { get; set; }
 
         [JsonPropertyName("updateURL")]
-        public string UpdateURL { get; set; }
+        public string? UpdateURL { get; set; }
 
         [JsonPropertyName("cancelURL")]
-        public string CancelURL { get; set; }
+        public string? CancelURL { get; set; }
 
         [JsonPropertyName("subscriptionId")]
-        public string SubscriptionId { get; set; }
+        public string? SubscriptionId { get; set; }
 
         [JsonPropertyName("productId")]
-        public string ProductId { get; set; }
+        public string? ProductId { get; set; }
 
         [JsonPropertyName("extend")]
         public bool Extend { get; set; }

Streetwriters.Common/Messages/CreateSubscriptionMessageV2.cs

@@ -28,7 +28,7 @@ namespace Streetwriters.Common.Messages
     public class CreateSubscriptionMessageV2
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("provider")]
         public SubscriptionProvider Provider { get; set; }
@@ -49,13 +49,13 @@ namespace Streetwriters.Common.Messages
         public long ExpiryTime { get; set; }
 
         [JsonPropertyName("orderId")]
-        public string OrderId { get; set; }
+        public string? OrderId { get; set; }
 
         [JsonPropertyName("subscriptionId")]
-        public string SubscriptionId { get; set; }
+        public string? SubscriptionId { get; set; }
 
         [JsonPropertyName("productId")]
-        public string ProductId { get; set; }
+        public string? ProductId { get; set; }
 
         [JsonPropertyName("timestamp")]
         public long Timestamp { get; set; }

Streetwriters.Common/Messages/DeleteSubscriptionMessage.cs

@@ -27,7 +27,7 @@ namespace Streetwriters.Common.Messages
     public class DeleteSubscriptionMessage
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("appId")]
         public ApplicationType AppId { get; set; }

Streetwriters.Common/Messages/DeleteUserMessage.cs

@@ -27,6 +27,6 @@ namespace Streetwriters.Common.Messages
     public class DeleteUserMessage
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
     }
 }

Streetwriters.Common/Messages/SendSSEMessage.cs

@@ -26,10 +26,10 @@ namespace Streetwriters.Common.Messages
     public class Message
     {
         [JsonPropertyName("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("data")]
-        public string Data { get; set; }
+        public string? Data { get; set; }
     }
     public class SendSSEMessage
     {
@@ -37,10 +37,10 @@ namespace Streetwriters.Common.Messages
         public bool SendToAll { get; set; }
 
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("message")]
-        public Message Message { get; set; }
+        public required Message Message { get; set; }
 
         [JsonPropertyName("originTokenId")]
         public string? OriginTokenId { get; set; }

Streetwriters.Common/Models/Client.cs

@@ -31,15 +31,15 @@ namespace Streetwriters.Common.Models
 {
     public class Client : IClient
     {
-        public string Id { get; set; }
-        public string Name { get; set; }
+        public required string Id { get; set; }
+        public required string Name { get; set; }
         public ApplicationType Type { get; set; }
         public ApplicationType AppId { get; set; }
-        public string SenderEmail { get; set; }
-        public string SenderName { get; set; }
-        public string EmailConfirmedRedirectURL { get; set; }
-        public string AccountRecoveryRedirectURL { get; set; }
+        public required string SenderEmail { get; set; }
+        public required string SenderName { get; set; }
+        public required string EmailConfirmedRedirectURL { get; set; }
+        public required string AccountRecoveryRedirectURL { get; set; }
 
-        public Func<string, Task> OnEmailConfirmed { get; set; }
+        public Func<string, Task>? OnEmailConfirmed { get; set; }
     }
 }

Streetwriters.Common/Models/EmailTemplate.cs

@@ -3,9 +3,9 @@ namespace Streetwriters.Common.Models
     public class EmailTemplate
     {
         public int? Id { get; set; }
-        public object Data { get; set; }
-        public string Subject { get; set; }
-        public string Html { get; set; }
-        public string Text { get; set; }
+        public object? Data { get; set; }
+        public required string Subject { get; set; }
+        public required string Html { get; set; }
+        public required string Text { get; set; }
     }
 }

Streetwriters.Common/Models/GetCustomerResponse.cs

@@ -10,12 +10,12 @@ namespace Streetwriters.Common.Models
     public partial class GetCustomerResponse : PaddleResponse
     {
         [JsonPropertyName("data")]
-        public PaddleCustomer Customer { get; set; }
+        public PaddleCustomer? Customer { get; set; }
     }
 
     public class PaddleCustomer
     {
         [JsonPropertyName("email")]
-        public string Email { get; set; }
+        public string? Email { get; set; }
     }
 }

Streetwriters.Common/Models/GetSubscriptionResponse.cs

@@ -10,7 +10,7 @@ namespace Streetwriters.Common.Models
     public partial class GetSubscriptionResponse : PaddleResponse
     {
         [JsonPropertyName("data")]
-        public Data Data { get; set; }
+        public Data? Data { get; set; }
     }
 
     public partial class Data
@@ -22,7 +22,7 @@ namespace Streetwriters.Common.Models
         // public string Status { get; set; }
 
         [JsonPropertyName("customer_id")]
-        public string CustomerId { get; set; }
+        public string? CustomerId { get; set; }
 
         // [JsonPropertyName("address_id")]
         // public string AddressId { get; set; }
@@ -64,7 +64,7 @@ namespace Streetwriters.Common.Models
         // public CurrentBillingPeriod CurrentBillingPeriod { get; set; }
 
         [JsonPropertyName("billing_cycle")]
-        public BillingCycle BillingCycle { get; set; }
+        public BillingCycle? BillingCycle { get; set; }
 
         // [JsonPropertyName("scheduled_change")]
         // public object ScheduledChange { get; set; }
@@ -76,7 +76,7 @@ namespace Streetwriters.Common.Models
         // public object CustomData { get; set; }
 
         [JsonPropertyName("management_urls")]
-        public ManagementUrls ManagementUrls { get; set; }
+        public ManagementUrls? ManagementUrls { get; set; }
 
         // [JsonPropertyName("discount")]
         // public object Discount { get; set; }
@@ -91,7 +91,7 @@ namespace Streetwriters.Common.Models
         public long Frequency { get; set; }
 
         [JsonPropertyName("interval")]
-        public string Interval { get; set; }
+        public string? Interval { get; set; }
     }
 
     // public partial class CurrentBillingPeriod
@@ -206,9 +206,9 @@ namespace Streetwriters.Common.Models
     public partial class ManagementUrls
     {
         [JsonPropertyName("update_payment_method")]
-        public Uri UpdatePaymentMethod { get; set; }
+        public Uri? UpdatePaymentMethod { get; set; }
 
         [JsonPropertyName("cancel")]
-        public Uri Cancel { get; set; }
+        public Uri? Cancel { get; set; }
     }
 }

Streetwriters.Common/Models/GetTransactionInvoiceResponse.cs

@@ -10,12 +10,12 @@ namespace Streetwriters.Common.Models
     public class GetTransactionInvoiceResponse : PaddleResponse
     {
         [JsonPropertyName("data")]
-        public Invoice Invoice { get; set; }
+        public Invoice? Invoice { get; set; }
     }
 
     public partial class Invoice
     {
         [JsonPropertyName("url")]
-        public string Url { get; set; }
+        public string? Url { get; set; }
     }
 }

Streetwriters.Common/Models/GetTransactionResponse.cs

@@ -10,6 +10,6 @@ namespace Streetwriters.Common.Models
     public partial class GetTransactionResponse : PaddleResponse
     {
         [JsonPropertyName("data")]
-        public TransactionV2 Transaction { get; set; }
+        public TransactionV2? Transaction { get; set; }
     }
 }

Streetwriters.Common/Models/GiftCard.cs

@@ -9,14 +9,14 @@ namespace Streetwriters.Common.Models
     {
         public GiftCard()
         {
-            Id = ObjectId.GenerateNewId().ToString();
+            Id = ObjectId.GenerateNewId();
         }
 
-        public string Code { get; set; }
-        public string OrderId { get; set; }
-        public string OrderIdType { get; set; }
-        public string ProductId { get; set; }
-        public string RedeemedBy { get; set; }
+        public required string Code { get; set; }
+        public required string OrderId { get; set; }
+        public required string OrderIdType { get; set; }
+        public required string ProductId { get; set; }
+        public string? RedeemedBy { get; set; }
         public long RedeemedAt { get; set; }
         public long Timestamp { get; set; }
         public long Term { get; set; }
@@ -24,6 +24,6 @@ namespace Streetwriters.Common.Models
         [BsonId]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
-        public string Id { get; set; }
+        public ObjectId Id { get; set; }
     }
 }

Streetwriters.Common/Models/ListPaymentsResponse.cs

@@ -9,7 +9,7 @@ namespace Streetwriters.Common.Models
         public bool Success { get; set; }
 
         [JsonPropertyName("response")]
-        public Payment[] Payments { get; set; }
+        public Payment[]? Payments { get; set; }
     }
 
     public partial class Payment
@@ -24,10 +24,10 @@ namespace Streetwriters.Common.Models
         public double Amount { get; set; }
 
         [JsonPropertyName("currency")]
-        public string Currency { get; set; }
+        public string? Currency { get; set; }
 
         [JsonPropertyName("payout_date")]
-        public string PayoutDate { get; set; }
+        public string? PayoutDate { get; set; }
 
         [JsonPropertyName("is_paid")]
         public short IsPaid { get; set; }
@@ -36,6 +36,6 @@ namespace Streetwriters.Common.Models
         public bool IsOneOffCharge { get; set; }
 
         [JsonPropertyName("receipt_url")]
-        public string ReceiptUrl { get; set; }
+        public string? ReceiptUrl { get; set; }
     }
 }

Streetwriters.Common/Models/ListTransactionsResponse.cs

@@ -9,31 +9,31 @@ namespace Streetwriters.Common.Models
         public bool Success { get; set; }
 
         [JsonPropertyName("response")]
-        public Transaction[] Transactions { get; set; }
+        public Transaction[]? Transactions { get; set; }
     }
 
     public partial class Transaction
     {
         [JsonPropertyName("order_id")]
-        public string OrderId { get; set; }
+        public string? OrderId { get; set; }
 
         [JsonPropertyName("checkout_id")]
-        public string CheckoutId { get; set; }
+        public string? CheckoutId { get; set; }
 
         [JsonPropertyName("amount")]
-        public string Amount { get; set; }
+        public string? Amount { get; set; }
 
         [JsonPropertyName("currency")]
-        public string Currency { get; set; }
+        public string? Currency { get; set; }
 
         [JsonPropertyName("status")]
-        public string Status { get; set; }
+        public string? Status { get; set; }
 
         [JsonPropertyName("created_at")]
-        public string CreatedAt { get; set; }
+        public string? CreatedAt { get; set; }
 
         [JsonPropertyName("passthrough")]
-        public object Passthrough { get; set; }
+        public object? Passthrough { get; set; }
 
         [JsonPropertyName("product_id")]
         public long ProductId { get; set; }
@@ -45,13 +45,13 @@ namespace Streetwriters.Common.Models
         public bool IsOneOff { get; set; }
 
         [JsonPropertyName("subscription")]
-        public PaddleSubscription Subscription { get; set; }
+        public PaddleSubscription? Subscription { get; set; }
 
         [JsonPropertyName("user")]
-        public PaddleTransactionUser User { get; set; }
+        public PaddleTransactionUser? User { get; set; }
 
         [JsonPropertyName("receipt_url")]
-        public string ReceiptUrl { get; set; }
+        public string? ReceiptUrl { get; set; }
     }
 
     public partial class PaddleSubscription
@@ -60,7 +60,7 @@ namespace Streetwriters.Common.Models
         public long SubscriptionId { get; set; }
 
         [JsonPropertyName("status")]
-        public string Status { get; set; }
+        public string? Status { get; set; }
     }
 
     public partial class PaddleTransactionUser
@@ -69,7 +69,7 @@ namespace Streetwriters.Common.Models
         public long UserId { get; set; }
 
         [JsonPropertyName("email")]
-        public string Email { get; set; }
+        public string? Email { get; set; }
 
         [JsonPropertyName("marketing_consent")]
         public bool MarketingConsent { get; set; }

Streetwriters.Common/Models/ListTransactionsResponseV2.cs

@@ -10,19 +10,19 @@ namespace Streetwriters.Common.Models
     public partial class ListTransactionsResponseV2 : PaddleResponse
     {
         [JsonPropertyName("data")]
-        public TransactionV2[] Transactions { get; set; }
+        public TransactionV2[]? Transactions { get; set; }
     }
 
     public partial class TransactionV2
     {
         [JsonPropertyName("id")]
-        public string Id { get; set; }
+        public string? Id { get; set; }
 
         [JsonPropertyName("status")]
-        public string Status { get; set; }
+        public string? Status { get; set; }
 
         [JsonPropertyName("customer_id")]
-        public string CustomerId { get; set; }
+        public string? CustomerId { get; set; }
 
         // [JsonPropertyName("address_id")]
         // public string AddressId { get; set; }
@@ -31,10 +31,10 @@ namespace Streetwriters.Common.Models
         // public object BusinessId { get; set; }
 
         [JsonPropertyName("custom_data")]
-        public Dictionary<string, string> CustomData { get; set; }
+        public Dictionary<string, string>? CustomData { get; set; }
 
         [JsonPropertyName("origin")]
-        public string Origin { get; set; }
+        public string? Origin { get; set; }
 
         // [JsonPropertyName("collection_mode")]
         // public string CollectionMode { get; set; }
@@ -49,10 +49,10 @@ namespace Streetwriters.Common.Models
         // public string InvoiceNumber { get; set; }
 
         [JsonPropertyName("billing_details")]
-        public BillingDetails BillingDetails { get; set; }
+        public BillingDetails? BillingDetails { get; set; }
 
         [JsonPropertyName("billing_period")]
-        public BillingPeriod BillingPeriod { get; set; }
+        public BillingPeriod? BillingPeriod { get; set; }
 
         // [JsonPropertyName("currency_code")]
         // public string CurrencyCode { get; set; }
@@ -70,10 +70,10 @@ namespace Streetwriters.Common.Models
         public DateTimeOffset? BilledAt { get; set; }
 
         [JsonPropertyName("items")]
-        public Item[] Items { get; set; }
+        public Item[]? Items { get; set; }
 
         [JsonPropertyName("details")]
-        public Details Details { get; set; }
+        public Details? Details { get; set; }
 
         // [JsonPropertyName("payments")]
         // public Payment[] Payments { get; set; }
@@ -88,7 +88,7 @@ namespace Streetwriters.Common.Models
         // public bool EnableCheckout { get; set; }
 
         [JsonPropertyName("payment_terms")]
-        public PaymentTerms PaymentTerms { get; set; }
+        public PaymentTerms? PaymentTerms { get; set; }
 
         // [JsonPropertyName("purchase_order_number")]
         // public string PurchaseOrderNumber { get; set; }
@@ -100,7 +100,7 @@ namespace Streetwriters.Common.Models
     public partial class PaymentTerms
     {
         [JsonPropertyName("interval")]
-        public string Interval { get; set; }
+        public string? Interval { get; set; }
 
         [JsonPropertyName("frequency")]
         public long Frequency { get; set; }
@@ -127,7 +127,7 @@ namespace Streetwriters.Common.Models
         // public TaxRatesUsed[] TaxRatesUsed { get; set; }
 
         [JsonPropertyName("totals")]
-        public Totals Totals { get; set; }
+        public Totals? Totals { get; set; }
 
         // [JsonPropertyName("adjusted_totals")]
         // public AdjustedTotals AdjustedTotals { get; set; }
@@ -139,7 +139,7 @@ namespace Streetwriters.Common.Models
         // public AdjustedTotals AdjustedPayoutTotals { get; set; }
 
         [JsonPropertyName("line_items")]
-        public LineItem[] LineItems { get; set; }
+        public LineItem[]? LineItems { get; set; }
     }
 
     public partial class Totals
@@ -175,7 +175,7 @@ namespace Streetwriters.Common.Models
         // public object Earnings { get; set; }
 
         [JsonPropertyName("currency_code")]
-        public string CurrencyCode { get; set; }
+        public string? CurrencyCode { get; set; }
     }
     // public partial class AdjustedTotals
     // {
@@ -225,10 +225,10 @@ namespace Streetwriters.Common.Models
     public partial class LineItem
     {
         [JsonPropertyName("id")]
-        public string Id { get; set; }
+        public string? Id { get; set; }
 
         [JsonPropertyName("price_id")]
-        public string PriceId { get; set; }
+        public string? PriceId { get; set; }
 
         // [JsonPropertyName("quantity")]
         // public long Quantity { get; set; }
@@ -247,7 +247,7 @@ namespace Streetwriters.Common.Models
 
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
         [JsonPropertyName("proration")]
-        public Proration Proration { get; set; }
+        public Proration? Proration { get; set; }
     }
 
     // public partial class Product
@@ -322,7 +322,7 @@ namespace Streetwriters.Common.Models
     public partial class Proration
     {
         [JsonPropertyName("billing_period")]
-        public BillingPeriod BillingPeriod { get; set; }
+        public BillingPeriod? BillingPeriod { get; set; }
     }
 
     // public partial class Totals
@@ -356,20 +356,20 @@ namespace Streetwriters.Common.Models
     public partial class Item
     {
         [JsonPropertyName("price")]
-        public Price Price { get; set; }
+        public Price? Price { get; set; }
 
         [JsonPropertyName("quantity")]
         public long Quantity { get; set; }
 
         [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
         [JsonPropertyName("proration")]
-        public Proration Proration { get; set; }
+        public Proration? Proration { get; set; }
     }
 
     public partial class Price
     {
         [JsonPropertyName("id")]
-        public string Id { get; set; }
+        public string? Id { get; set; }
 
         // [JsonPropertyName("description")]
         // public string Description { get; set; }
@@ -378,7 +378,7 @@ namespace Streetwriters.Common.Models
         // public TypeEnum Type { get; set; }
 
         [JsonPropertyName("name")]
-        public string Name { get; set; }
+        public string? Name { get; set; }
 
         // [JsonPropertyName("product_id")]
         // public string ProductId { get; set; }
@@ -500,7 +500,7 @@ namespace Streetwriters.Common.Models
         public long PerPage { get; set; }
 
         [JsonPropertyName("next")]
-        public Uri Next { get; set; }
+        public Uri? Next { get; set; }
 
         [JsonPropertyName("has_more")]
         public bool HasMore { get; set; }

Streetwriters.Common/Models/ListUsersResponse.cs

@@ -9,7 +9,7 @@ namespace Streetwriters.Common.Models
         public bool Success { get; set; }
 
         [JsonPropertyName("response")]
-        public PaddleUser[] Users { get; set; }
+        public PaddleUser[]? Users { get; set; }
     }
 
     public class PaddleUser
@@ -24,22 +24,22 @@ namespace Streetwriters.Common.Models
         public long UserId { get; set; }
 
         [JsonPropertyName("user_email")]
-        public string UserEmail { get; set; }
+        public string? UserEmail { get; set; }
 
         [JsonPropertyName("marketing_consent")]
         public bool MarketingConsent { get; set; }
 
         [JsonPropertyName("update_url")]
-        public string UpdateUrl { get; set; }
+        public string? UpdateUrl { get; set; }
 
         [JsonPropertyName("cancel_url")]
-        public string CancelUrl { get; set; }
+        public string? CancelUrl { get; set; }
 
         [JsonPropertyName("state")]
-        public string State { get; set; }
+        public string? State { get; set; }
 
         [JsonPropertyName("signup_date")]
-        public string SignupDate { get; set; }
+        public string? SignupDate { get; set; }
 
         [JsonPropertyName("quantity")]
         public long Quantity { get; set; }

Streetwriters.Common/Models/MFAConfig.cs

@@ -22,8 +22,8 @@ namespace Streetwriters.Common.Models
     public class MFAConfig
     {
         public bool IsEnabled { get; set; }
-        public string PrimaryMethod { get; set; }
-        public string SecondaryMethod { get; set; }
+        public required string PrimaryMethod { get; set; }
+        public string? SecondaryMethod { get; set; }
         public int RemainingValidCodes { get; set; }
     }
 }

Streetwriters.Common/Models/Offer.cs

@@ -29,25 +29,25 @@ using Streetwriters.Common.Interfaces;
 
 namespace Streetwriters.Common.Models
 {
-    public class Offer : IOffer
+    public class Offer
     {
         public Offer()
         {
-            Id = ObjectId.GenerateNewId().ToString();
+            Id = ObjectId.GenerateNewId();
         }
 
         [BsonId]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonPropertyName("id")]
-        public string Id { get; set; }
+        public ObjectId Id { get; set; }
 
         [JsonPropertyName("appId")]
         public ApplicationType AppId { get; set; }
 
         [JsonPropertyName("promoCode")]
-        public string PromoCode { get; set; }
+        public required string PromoCode { get; set; }
 
         [JsonPropertyName("codes")]
-        public PromoCode[] Codes { get; set; }
+        public required PromoCode[] Codes { get; set; }
     }
 }

Streetwriters.Common/Models/PaddleResponse.cs

@@ -10,7 +10,7 @@ namespace Streetwriters.Common.Models
     public partial class PaddleResponse
     {
         [JsonPropertyName("error")]
-        public PaddleError Error { get; set; }
+        public PaddleError? Error { get; set; }
     }
 
     public class PaddleError

Streetwriters.Common/Models/PromoCode.cs

@@ -35,6 +35,6 @@ namespace Streetwriters.Common.Models
         public SubscriptionProvider Provider { get; set; }
 
         [JsonPropertyName("code")]
-        public string Code { get; set; }
+        public required string Code { get; set; }
     }
 }

Streetwriters.Common/Models/RefundPaymentResponse.cs

@@ -9,7 +9,7 @@ namespace Streetwriters.Common.Models
         public bool Success { get; set; }
 
         [JsonPropertyName("response")]
-        public Refund Refund { get; set; }
+        public required Refund Refund { get; set; }
     }
 
     public partial class Refund

Streetwriters.Common/Models/Response.cs

@@ -30,6 +30,6 @@ namespace Streetwriters.Common.Models
         public bool Success { get; set; }
         public int StatusCode { get; set; }
         [JsonIgnore]
-        public HttpContent Content { get; set; }
+        public HttpContent? Content { get; set; }
     }
 }

Streetwriters.Common/Models/Subscription.cs

@@ -65,8 +65,12 @@ namespace Streetwriters.Common.Models
 
         [BsonRepresentation(BsonType.Int32)]
         [JsonPropertyName("type")]
-        [Obsolete("Use SubscriptionPlan and SubscriptionStatus instead.")]
-        public SubscriptionType Type { get; set; }
+        public SubscriptionType Type
+        {
+            get;
+            [Obsolete("Use SubscriptionPlan and SubscriptionStatus instead.")]
+            set;
+        }
 
         [JsonPropertyName("cancelURL")]
         public string? CancelURL { get; set; }

Streetwriters.Common/Models/SubscriptionPreviewResponse.cs

@@ -10,40 +10,40 @@ namespace Streetwriters.Common.Models
     public partial class SubscriptionPreviewResponse : PaddleResponse
     {
         [JsonPropertyName("data")]
-        public SubscriptionPreviewData Data { get; set; }
+        public SubscriptionPreviewData? Data { get; set; }
     }
 
     public partial class SubscriptionPreviewData
     {
         [JsonPropertyName("currency_code")]
-        public string CurrencyCode { get; set; }
+        public string? CurrencyCode { get; set; }
 
         [JsonPropertyName("billing_cycle")]
-        public BillingCycle BillingCycle { get; set; }
+        public BillingCycle? BillingCycle { get; set; }
 
         [JsonPropertyName("update_summary")]
-        public UpdateSummary UpdateSummary { get; set; }
+        public UpdateSummary? UpdateSummary { get; set; }
 
         [JsonPropertyName("immediate_transaction")]
-        public TransactionV2 ImmediateTransaction { get; set; }
+        public TransactionV2? ImmediateTransaction { get; set; }
 
         [JsonPropertyName("next_transaction")]
-        public TransactionV2 NextTransaction { get; set; }
+        public TransactionV2? NextTransaction { get; set; }
 
         [JsonPropertyName("recurring_transaction_details")]
-        public Details RecurringTransactionDetails { get; set; }
+        public Details? RecurringTransactionDetails { get; set; }
     }
 
     public partial class UpdateSummary
     {
         [JsonPropertyName("charge")]
-        public UpdateSummaryItem Charge { get; set; }
+        public UpdateSummaryItem? Charge { get; set; }
 
         [JsonPropertyName("credit")]
-        public UpdateSummaryItem Credit { get; set; }
+        public UpdateSummaryItem? Credit { get; set; }
 
         [JsonPropertyName("result")]
-        public UpdateSummaryItem Result { get; set; }
+        public UpdateSummaryItem? Result { get; set; }
     }
 
     public partial class UpdateSummaryItem

Streetwriters.Common/Models/UserModel.cs

@@ -24,13 +24,13 @@ namespace Streetwriters.Common.Models
     public class UserModel
     {
         [JsonPropertyName("id")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("email")]
-        public string Email { get; set; }
+        public required string Email { get; set; }
 
         [JsonPropertyName("phoneNumber")]
-        public string PhoneNumber { get; set; }
+        public string? PhoneNumber { get; set; }
 
         [JsonPropertyName("isEmailConfirmed")]
         public bool IsEmailConfirmed { get; set; }
@@ -39,7 +39,7 @@ namespace Streetwriters.Common.Models
         public bool MarketingConsent { get; set; }
 
         [JsonPropertyName("mfa")]
-        public MFAConfig MFA { get; set; }
+        public required MFAConfig MFA { get; set; }
     }
 
 }

Streetwriters.Common/Servers.cs

@@ -30,16 +30,16 @@ namespace Streetwriters.Common
 {
     public class Server
     {
-        public Server(string originCertPath = null, string originCertKeyPath = null)
+        public Server(string? originCertPath = null, string? originCertKeyPath = null)
         {
             if (!string.IsNullOrEmpty(originCertPath) && !string.IsNullOrEmpty(originCertKeyPath))
                 this.SSLCertificate = X509Certificate2.CreateFromPemFile(originCertPath, originCertKeyPath);
         }
-        public string Id { get; set; }
+        public string? Id { get; set; }
         public int Port { get; set; }
-        public string Hostname { get; set; }
-        public Uri PublicURL { get; set; }
-        public X509Certificate2 SSLCertificate { get; }
+        public required string Hostname { get; set; }
+        public Uri? PublicURL { get; set; }
+        public X509Certificate2? SSLCertificate { get; }
         public bool IsSecure { get => this.SSLCertificate != null; }
 
         public override string ToString()
@@ -93,14 +93,14 @@ namespace Streetwriters.Common
         public static Server NotesnookAPI { get; } = new(Constants.NOTESNOOK_CERT_PATH, Constants.NOTESNOOK_CERT_KEY_PATH)
         {
             Port = Constants.NOTESNOOK_SERVER_PORT,
-            Hostname = Constants.NOTESNOOK_SERVER_HOST,
+            Hostname = Constants.NOTESNOOK_SERVER_HOST ?? "localhost",
             Id = "notesnook-sync"
         };
 
         public static Server MessengerServer { get; } = new(Constants.SSE_CERT_PATH, Constants.SSE_CERT_KEY_PATH)
         {
             Port = Constants.SSE_SERVER_PORT,
-            Hostname = Constants.SSE_SERVER_HOST,
+            Hostname = Constants.SSE_SERVER_HOST ?? "localhost",
             Id = "sse"
         };
 
@@ -108,14 +108,14 @@ namespace Streetwriters.Common
         {
             PublicURL = Constants.IDENTITY_SERVER_URL,
             Port = Constants.IDENTITY_SERVER_PORT,
-            Hostname = Constants.IDENTITY_SERVER_HOST,
+            Hostname = Constants.IDENTITY_SERVER_HOST ?? "localhost",
             Id = "auth"
         };
 
         public static Server SubscriptionServer { get; } = new(Constants.SUBSCRIPTIONS_CERT_PATH, Constants.SUBSCRIPTIONS_CERT_KEY_PATH)
         {
             Port = Constants.SUBSCRIPTIONS_SERVER_PORT,
-            Hostname = Constants.SUBSCRIPTIONS_SERVER_HOST,
+            Hostname = Constants.SUBSCRIPTIONS_SERVER_HOST ?? "localhost",
             Id = "subscription"
         };
     }

Streetwriters.Common/Services/EmailSender.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Text;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
 using MailKit.Net.Smtp;
 using MimeKit;
 using MimeKit.Cryptography;
@@ -16,13 +17,19 @@ namespace Streetwriters.Common.Services
     public class EmailSender : IEmailSender, IAsyncDisposable
     {
         private readonly SmtpClient mailClient = new();
+        private readonly ILogger<EmailSender> logger;
+
+        public EmailSender(ILogger<EmailSender> logger)
+        {
+            this.logger = logger;
+        }
 
         public async Task SendEmailAsync(
             string email,
             EmailTemplate template,
             IClient client,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
+            GnuPGContext? gpgContext = null,
+            Dictionary<string, byte[]>? attachments = null
         )
         {
             if (!mailClient.IsConnected)
@@ -67,12 +74,12 @@ namespace Streetwriters.Common.Services
             await mailClient.SendAsync(message);
         }
 
-        private static async Task<MimeEntity> GetEmailBodyAsync(
+        private async Task<MimeEntity> GetEmailBodyAsync(
             EmailTemplate template,
             IClient client,
             MailboxAddress sender,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
+            GnuPGContext? gpgContext = null,
+            Dictionary<string, byte[]>? attachments = null
         )
         {
             var builder = new BodyBuilder();
@@ -120,7 +127,7 @@ namespace Streetwriters.Common.Services
             }
             catch (Exception ex)
             {
-                await Slogger<EmailSender>.Error("GetEmailBodyAsync", ex.ToString());
+                logger.LogError(ex, "Failed to get email body");
                 return builder.ToMessageBody();
             }
         }

Streetwriters.Common/Services/PaddleBillingService.cs

@@ -129,7 +129,7 @@ namespace Streetwriters.Common.Services
         public async Task<GetCustomerResponse?> FindCustomerFromTransactionAsync(string transactionId)
         {
             var transaction = await GetTransactionAsync(transactionId);
-            if (transaction == null) return null;
+            if (transaction?.Transaction?.CustomerId == null) return null;
             var url = $"{PADDLE_BASE_URI}/customers/{transaction.Transaction.CustomerId}";
             var response = await httpClient.GetFromJsonAsync<GetCustomerResponse>(url);
             return response;

Streetwriters.Common/Services/PaddleService.cs

@@ -18,7 +18,7 @@ namespace Streetwriters.Common.Services
 
         HttpClient httpClient = new HttpClient();
 
-        public async Task<ListUsersResponse> ListUsersAsync(
+        public async Task<ListUsersResponse?> ListUsersAsync(
             string subscriptionId,
             int results
         )
@@ -41,7 +41,7 @@ namespace Streetwriters.Common.Services
             return await response.Content.ReadFromJsonAsync<ListUsersResponse>();
         }
 
-        public async Task<ListPaymentsResponse> ListPaymentsAsync(
+        public async Task<ListPaymentsResponse?> ListPaymentsAsync(
             string subscriptionId,
             long planId
         )
@@ -66,7 +66,7 @@ namespace Streetwriters.Common.Services
             return await response.Content.ReadFromJsonAsync<ListPaymentsResponse>();
         }
 
-        public async Task<ListTransactionsResponse> ListTransactionsAsync(
+        public async Task<ListTransactionsResponse?> ListTransactionsAsync(
             string subscriptionId
         )
         {
@@ -86,7 +86,7 @@ namespace Streetwriters.Common.Services
             return await response.Content.ReadFromJsonAsync<ListTransactionsResponse>();
         }
 
-        public async Task<PaddleTransactionUser> FindUserFromOrderAsync(string orderId)
+        public async Task<PaddleTransactionUser?> FindUserFromOrderAsync(string orderId)
         {
             var url = $"{PADDLE_BASE_URI}/2.0/order/{orderId}/transactions";
             var httpClient = new HttpClient();
@@ -101,7 +101,7 @@ namespace Streetwriters.Common.Services
                 )
             );
             var transactions = await response.Content.ReadFromJsonAsync<ListTransactionsResponse>();
-            if (transactions.Transactions.Length == 0) return null;
+            if (transactions?.Transactions == null || transactions.Transactions.Length == 0) return null;
             return transactions.Transactions[0].User;
         }
 
@@ -123,7 +123,7 @@ namespace Streetwriters.Common.Services
             );
 
             var refundResponse = await response.Content.ReadFromJsonAsync<RefundPaymentResponse>();
-            return refundResponse.Success;
+            return refundResponse?.Success ?? false;
         }
 
         public async Task<bool> CancelSubscriptionAsync(string subscriptionId)