identity: log on email confirmation

* docker: update .env and docker-compose.yml for SMTP and new services

* docker: add external MinIO, MongoDB, and GarageHQ examples with setup instructions

* docs: update README with dotenv docker setup instructions and configuration redirect for external services

* docker: update service dependency condition for setup-themes in docker-compose.yml

* chore: update example URLs in .env for public services

.env

@@ -12,21 +12,26 @@ NOTESNOOK_API_SECRET=
 # Possible values: true/false
 DISABLE_SIGNUPS=false
 
-### SMTP Configuration ###
-# SMTP Configuration is required for sending emails for password reset, 2FA emails etc. You can get SMTP settings from your email provider.
+### SMTP Configuration (optional) ###
+# ================================================================================
+# WARNING: If SMTP is not configured, email 2FA and password-reset emails will
+# NOT work. Newly created users MUST change 2FA to an Authenticator app:
+#   Settings > Auth > Change 2FA method
+# ================================================================================
+# SMTP is used for sending emails for password reset, 2FA emails etc. You can get SMTP settings from your email provider.
 
 # Description: Username for the SMTP connection (most time it is the email address of your account). Check your email provider's documentation to get the appropriate value.
-# Required: yes
+# Required: no
 SMTP_USERNAME=
 # Description: Password for the SMTP connection. Check your email provider's documentation to get the appropriate value.
-# Required: yes
+# Required: no
 SMTP_PASSWORD=
 # Description: Host on which the the SMTP connection is running. Check your email provider's documentation to get the appropriate value.
-# Required: yes
+# Required: no
 # Example: smtp.gmail.com
 SMTP_HOST=
 # Description: Port on which the the SMTP connection is running. Check your email provider's documentation to get the appropriate value.
-# Required: yes
+# Required: no
 # Example: 465
 SMTP_PORT=
 
@@ -42,10 +47,16 @@ TWILIO_AUTH_TOKEN=
 TWILIO_SERVICE_SID=
 
 # Description: Add the origins for which you want to allow CORS. Leave it empty to allow all origins to access your server. If you want to allow multiple origins, seperate each origin with a comma.
+# Note: this value is passed to the server as NOTESNOOK_CORS internally.
 # Required: no
 # Example: https://app.notesnook.com,http://localhost:3000
 NOTESNOOK_CORS_ORIGINS=
 
+# Description: Add known proxies for incoming HTTP requests
+# Required: no
+# Example: 192.168.1.2,192.168.1.3
+KNOWN_PROXIES=
+
 # Description: This is the public URL for the web app, and is used by the backend for creating redirect URLs (e.g. after email confirmation etc). 
 # Note: the URL has no slashes at the end
 # Required: yes
@@ -53,16 +64,20 @@ NOTESNOOK_CORS_ORIGINS=
 NOTESNOOK_APP_PUBLIC_URL=https://app.notesnook.com
 # Description: This is the public URL for the monograph frontend.
 # Required: yes
-# Example: https://monogr.ph
+# Example: https://monogr.yourdomain.com
 MONOGRAPH_PUBLIC_URL=http://localhost:6264
 # Description: This is the public URL for the Authentication server. Used for generating email confirmation & password reset URLs.
 # Required: yes
-# Example: https://auth.streetwriters.co
+# Example: https://auth.yourdomain.com
 AUTH_SERVER_PUBLIC_URL=http://localhost:8264
 # Description: This is the public URL for the S3 attachments server (minio). It'll be used by the Notesnook clients for uploading/downloading attachments.
 # Required: yes
-# Example: https://attachments.notesnook.com
+# Example: https://attachments.yourdomain.com
 ATTACHMENTS_SERVER_PUBLIC_URL=http://localhost:9000
+# Description: This is the public URL for the Notesnook sync API (browser-reachable; used by Monograph).
+# Required: yes
+# Example: https://api.yourdomain.com
+NOTESNOOK_API_PUBLIC_URL=http://localhost:5264
 
 # Description: Custom username for the root Minio account. Minio is used for storing your attachments. This must be greater than 3 characters in length.
 # Required: no
@@ -70,3 +85,15 @@ MINIO_ROOT_USER=
 # Description: Custom password for the root Minio account. Minio is used for storing your attachments. This must be greater than 8 characters in length.
 # Required: no
 MINIO_ROOT_PASSWORD=
+
+# Optional: only needed when using `docker compose --profile extras up`
+# Description: Public URL for the CORS proxy service
+CORS_PROXY_PUBLIC_URL=http://localhost:3001
+# Description: Allowed origins for the CORS proxy (* for all origins)
+CORS_PROXY_ALLOWED_ORIGINS=*
+# Description: Public URL for the Inbox API service
+INBOX_API_PUBLIC_URL=http://localhost:5181
+# Description: Public URL for the Themes server
+THEMES_SERVER_PUBLIC_URL=http://localhost:9200
+# Description: Git repository URL for the Themes server
+THEMES_REPO_URL=https://github.com/streetwriters/notesnook-themes.git

.github/workflows/publish.yml

@@ -10,8 +10,9 @@
 name: Publish Docker images
 
 on:
-  release:
-    types: [published]
+  push:
+    tags:
+      - "v*"
 
 jobs:
   push_to_registry:
@@ -22,12 +23,23 @@ jobs:
         repos:
           - image: streetwriters/notesnook-sync
             file: ./Notesnook.API/Dockerfile
+            context: .
+
+          - image: streetwriters/cors-proxy
+            file: ./cors-proxy/Dockerfile
+            context: ./cors-proxy/
 
           - image: streetwriters/identity
             file: ./Streetwriters.Identity/Dockerfile
+            context: .
 
           - image: streetwriters/sse
             file: ./Streetwriters.Messenger/Dockerfile
+            context: .
+
+          - image: streetwriters/notesnook-inbox
+            file: ./Notesnook.Inbox.API/Dockerfile
+            context: ./Notesnook.Inbox.API/
     permissions:
       packages: write
       contents: read
@@ -41,7 +53,7 @@ jobs:
       - name: Docker Setup Buildx
         uses: docker/setup-buildx-action@v3
         with:
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64
 
       - name: Log in to Docker Hub
         uses: docker/login-action@v3
@@ -70,10 +82,10 @@ jobs:
         id: push
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: ${{ matrix.repos.context }}
           file: ${{ matrix.repos.file }}
           push: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           cache-from: ${{ matrix.repos.image }}:latest
 

.vscode/launch.json

@@ -9,7 +9,7 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-notesnook",
-      "program": "bin/Debug/net8.0/Notesnook.API.dll",
+      "program": "bin/Debug/net9.0/Notesnook.API.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Notesnook.API",
       "stopAtEntry": false,
@@ -24,7 +24,7 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-identity",
-      "program": "bin/Debug/net8.0/Streetwriters.Identity.dll",
+      "program": "bin/Debug/net9.0/Streetwriters.Identity.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Identity",
       "stopAtEntry": false,
@@ -39,7 +39,7 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-messenger",
-      "program": "bin/Debug/net8.0/Streetwriters.Messenger.dll",
+      "program": "bin/Debug/net9.0/Streetwriters.Messenger.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Messenger",
       "stopAtEntry": false,

Notesnook.API/Accessors/SyncItemsRepositoryAccessor.cs

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
@@ -41,8 +42,13 @@ namespace Notesnook.API.Accessors
         public SyncItemsRepository Colors { get; }
         public SyncItemsRepository Vaults { get; }
         public SyncItemsRepository Tags { get; }
+        public SyncItemsRepository InboxItemsHistory { get; }
         public Repository<UserSettings> UsersSettings { get; }
         public Repository<Monograph> Monographs { get; }
+        public Repository<InboxApiKey> InboxApiKey { get; }
+        public Repository<InboxSyncItem> InboxItems { get; }
+        public Repository<SyncDevice> SyncDevices { get; }
+        public Repository<DeviceIdsChunk> DeviceIdsChunks { get; }
 
         public SyncItemsRepositoryAccessor(IDbContext dbContext,
 
@@ -70,23 +76,36 @@ namespace Notesnook.API.Accessors
         IMongoCollection<SyncItem> vaults,
         [FromKeyedServices(Collections.TagsKey)]
         IMongoCollection<SyncItem> tags,
+        [FromKeyedServices(Collections.InboxItemsHistoryKey)]
+        IMongoCollection<SyncItem> inboxItemsHistory,
 
-        Repository<UserSettings> usersSettings, Repository<Monograph> monographs)
+        Repository<UserSettings> usersSettings,
+        Repository<Monograph> monographs,
+        Repository<InboxApiKey> inboxApiKey,
+        Repository<InboxSyncItem> inboxItems,
+        Repository<SyncDevice> syncDevices,
+        Repository<DeviceIdsChunk> deviceIdsChunks,
+        ILogger<SyncItemsRepository> logger)
         {
             UsersSettings = usersSettings;
             Monographs = monographs;
-            Notebooks = new SyncItemsRepository(dbContext, notebooks);
-            Notes = new SyncItemsRepository(dbContext, notes);
-            Contents = new SyncItemsRepository(dbContext, content);
-            Settings = new SyncItemsRepository(dbContext, settings);
-            LegacySettings = new SyncItemsRepository(dbContext, legacySettings);
-            Attachments = new SyncItemsRepository(dbContext, attachments);
-            Shortcuts = new SyncItemsRepository(dbContext, shortcuts);
-            Reminders = new SyncItemsRepository(dbContext, reminders);
-            Relations = new SyncItemsRepository(dbContext, relations);
-            Colors = new SyncItemsRepository(dbContext, colors);
-            Vaults = new SyncItemsRepository(dbContext, vaults);
-            Tags = new SyncItemsRepository(dbContext, tags);
+            InboxApiKey = inboxApiKey;
+            InboxItems = inboxItems;
+            SyncDevices = syncDevices;
+            DeviceIdsChunks = deviceIdsChunks;
+            Notebooks = new SyncItemsRepository(dbContext, notebooks, logger);
+            Notes = new SyncItemsRepository(dbContext, notes, logger);
+            Contents = new SyncItemsRepository(dbContext, content, logger);
+            Settings = new SyncItemsRepository(dbContext, settings, logger);
+            LegacySettings = new SyncItemsRepository(dbContext, legacySettings, logger);
+            Attachments = new SyncItemsRepository(dbContext, attachments, logger);
+            Shortcuts = new SyncItemsRepository(dbContext, shortcuts, logger);
+            Reminders = new SyncItemsRepository(dbContext, reminders, logger);
+            Relations = new SyncItemsRepository(dbContext, relations, logger);
+            Colors = new SyncItemsRepository(dbContext, colors, logger);
+            Vaults = new SyncItemsRepository(dbContext, vaults, logger);
+            Tags = new SyncItemsRepository(dbContext, tags, logger);
+            InboxItemsHistory = new SyncItemsRepository(dbContext, inboxItemsHistory, logger);
         }
     }
 }

Notesnook.API/Authorization/InboxApiKeyAuthenticationHandler.cs

@@ -0,0 +1,101 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+using System.Security.Claims;
+using System.Text.Encodings.Web;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Notesnook.API.Models;
+using Streetwriters.Data.Repositories;
+
+namespace Notesnook.API.Authorization
+{
+    public static class InboxApiKeyAuthenticationDefaults
+    {
+        public const string AuthenticationScheme = "InboxApiKey";
+    }
+
+    public class InboxApiKeyAuthenticationSchemeOptions : AuthenticationSchemeOptions
+    {
+    }
+
+    public class InboxApiKeyAuthenticationHandler : AuthenticationHandler<InboxApiKeyAuthenticationSchemeOptions>
+    {
+        private readonly Repository<InboxApiKey> _inboxApiKeyRepository;
+
+        public InboxApiKeyAuthenticationHandler(
+            IOptionsMonitor<InboxApiKeyAuthenticationSchemeOptions> options,
+            ILoggerFactory logger,
+            UrlEncoder encoder,
+            Repository<InboxApiKey> inboxApiKeyRepository)
+            : base(options, logger, encoder)
+        {
+            _inboxApiKeyRepository = inboxApiKeyRepository;
+        }
+
+        protected override async Task<AuthenticateResult> HandleAuthenticateAsync()
+        {
+            if (!Request.Headers.ContainsKey("Authorization"))
+            {
+                return AuthenticateResult.Fail("Missing Authorization header");
+            }
+
+            var apiKey = Request.Headers["Authorization"].ToString().Trim();
+            if (string.IsNullOrEmpty(apiKey))
+            {
+                return AuthenticateResult.Fail("Missing API key");
+            }
+
+            try
+            {
+                var inboxApiKey = await _inboxApiKeyRepository.FindOneAsync(k => k.Key == apiKey);
+                if (inboxApiKey == null)
+                {
+                    return AuthenticateResult.Fail("Invalid API key");
+                }
+
+                if (inboxApiKey.ExpiryDate > 0 && DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() > inboxApiKey.ExpiryDate)
+                {
+                    return AuthenticateResult.Fail("API key has expired");
+                }
+
+                inboxApiKey.LastUsedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+                await _inboxApiKeyRepository.UpsertAsync(inboxApiKey, k => k.Key == apiKey);
+
+                var claims = new[]
+                {
+                    new Claim("sub", inboxApiKey.UserId),
+                };
+                var identity = new ClaimsIdentity(claims, Scheme.Name);
+                var principal = new ClaimsPrincipal(identity);
+                var ticket = new AuthenticationTicket(principal, Scheme.Name);
+
+                return AuthenticateResult.Success(ticket);
+            }
+            catch (Exception ex)
+            {
+                Logger.LogError(ex, "Error validating inbox API key");
+                return AuthenticateResult.Fail("Error validating API key");
+            }
+        }
+    }
+}

Notesnook.API/Authorization/ProUserRequirement.cs

@@ -1,63 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Claims;
-using System.Text.Json;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-
-namespace Notesnook.API.Authorization
-{
-    public class ProUserRequirement : AuthorizationHandler<ProUserRequirement>, IAuthorizationRequirement
-    {
-        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
-        {
-            ["/s3"] = "upload attachments",
-            ["/s3/multipart"] = "upload attachments",
-        };
-        private readonly string[] allowedClaims = ["trial", "premium", "premium_canceled"];
-        protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, ProUserRequirement requirement)
-        {
-            PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
-            var isProOrTrial = context.User.Claims.Any((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
-            if (isProOrTrial) context.Succeed(requirement);
-            else
-            {
-                var phrase = "continue";
-                foreach (var item in pathErrorPhraseMap)
-                {
-                    if (path != null && path.StartsWithSegments(item.Key))
-                        phrase = item.Value;
-                }
-                var error = $"Please upgrade to Pro to {phrase}.";
-                context.Fail(new AuthorizationFailureReason(this, error));
-            }
-            return Task.CompletedTask;
-        }
-
-        public override Task HandleAsync(AuthorizationHandlerContext context)
-        {
-            return this.HandleRequirementAsync(context, this);
-        }
-    }
-}

Notesnook.API/Authorization/SyncRequirement.cs

@@ -1,102 +1,102 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Collections.Generic;
-using System.Linq;
-using System.Security.Claims;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Authorization.Policy;
-using Microsoft.AspNetCore.Http;
-
-namespace Notesnook.API.Authorization
-{
-    public class SyncRequirement : AuthorizationHandler<SyncRequirement>, IAuthorizationRequirement
-    {
-        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
-        {
-            ["/sync/attachments"] = "use attachments",
-            ["/sync"] = "sync your notes",
-            ["/hubs/sync"] = "sync your notes",
-            ["/hubs/sync/v2"] = "sync your notes",
-            ["/monographs"] = "publish monographs"
-        };
-
-        protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, SyncRequirement requirement)
-        {
-            PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
-            var result = this.IsAuthorized(context.User, path);
-            if (result.Succeeded) context.Succeed(requirement);
-            else if (result.AuthorizationFailure.FailureReasons.Any())
-                context.Fail(result.AuthorizationFailure.FailureReasons.First());
-            else context.Fail();
-
-            return Task.CompletedTask;
-        }
-
-        public PolicyAuthorizationResult IsAuthorized(ClaimsPrincipal User, PathString requestPath)
-        {
-            var id = User.FindFirstValue("sub");
-
-            if (string.IsNullOrEmpty(id))
-            {
-                var reason = new[]
-                {
-                    new AuthorizationFailureReason(this, "Invalid token.")
-                };
-                return PolicyAuthorizationResult.Forbid(AuthorizationFailure.Failed(reason));
-            }
-
-            var hasSyncScope = User.HasClaim("scope", "notesnook.sync");
-            var isInAudience = User.HasClaim("aud", "notesnook");
-            var hasRole = User.HasClaim("role", "notesnook");
-
-            var isEmailVerified = User.HasClaim("verified", "true");
-
-            if (!isEmailVerified)
-            {
-                var phrase = "continue";
-
-                foreach (var item in pathErrorPhraseMap)
-                {
-                    if (requestPath != null && requestPath.StartsWithSegments(item.Key))
-                        phrase = item.Value;
-                }
-
-                var error = $"Please confirm your email to {phrase}.";
-                var reason = new[]
-                {
-                    new AuthorizationFailureReason(this, error)
-                };
-                return PolicyAuthorizationResult.Forbid(AuthorizationFailure.Failed(reason));
-                //  context.Fail(new AuthorizationFailureReason(this, error));
-            }
-
-            if (hasSyncScope && isInAudience && hasRole && isEmailVerified)
-                return PolicyAuthorizationResult.Success(); //(requirement);
-            return PolicyAuthorizationResult.Forbid();
-        }
-
-        public override Task HandleAsync(AuthorizationHandlerContext context)
-        {
-            return this.HandleRequirementAsync(context, this);
-        }
-
-    }
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Authorization.Policy;
+using Microsoft.AspNetCore.Http;
+
+namespace Notesnook.API.Authorization
+{
+    public class SyncRequirement : AuthorizationHandler<SyncRequirement>, IAuthorizationRequirement
+    {
+        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
+        {
+            ["/sync/attachments"] = "use attachments",
+            ["/sync"] = "sync your notes",
+            ["/hubs/sync"] = "sync your notes",
+            ["/hubs/sync/v2"] = "sync your notes",
+            ["/monographs"] = "publish monographs"
+        };
+
+        protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, SyncRequirement requirement)
+        {
+            PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
+            var result = this.IsAuthorized(context.User, path);
+            if (result.Succeeded) context.Succeed(requirement);
+            else if (result.AuthorizationFailure?.FailureReasons.Any() == true)
+                context.Fail(result.AuthorizationFailure.FailureReasons.First());
+            else context.Fail();
+
+            return Task.CompletedTask;
+        }
+
+        public PolicyAuthorizationResult IsAuthorized(ClaimsPrincipal? User, PathString requestPath)
+        {
+            var id = User?.FindFirstValue("sub");
+
+            if (string.IsNullOrEmpty(id))
+            {
+                var reason = new[]
+                {
+                    new AuthorizationFailureReason(this, "Invalid token.")
+                };
+                return PolicyAuthorizationResult.Forbid(AuthorizationFailure.Failed(reason));
+            }
+
+            var hasSyncScope = User?.HasClaim("scope", "notesnook.sync") ?? false;
+            var isInAudience = User?.HasClaim("aud", "notesnook") ?? false;
+            var hasRole = User?.HasClaim("role", "notesnook") ?? false;
+
+            var isEmailVerified = User?.HasClaim("verified", "true") ?? false;
+
+            if (!isEmailVerified)
+            {
+                var phrase = "continue";
+
+                foreach (var item in pathErrorPhraseMap)
+                {
+                    if (requestPath != null && requestPath.StartsWithSegments(item.Key))
+                        phrase = item.Value;
+                }
+
+                var error = $"Please confirm your email to {phrase}.";
+                var reason = new[]
+                {
+                    new AuthorizationFailureReason(this, error)
+                };
+                return PolicyAuthorizationResult.Forbid(AuthorizationFailure.Failed(reason));
+                //  context.Fail(new AuthorizationFailureReason(this, error));
+            }
+
+            if (hasSyncScope && isInAudience && hasRole && isEmailVerified)
+                return PolicyAuthorizationResult.Success(); //(requirement);
+            return PolicyAuthorizationResult.Forbid();
+        }
+
+        public override Task HandleAsync(AuthorizationHandlerContext context)
+        {
+            return this.HandleRequirementAsync(context, this);
+        }
+
+    }
 }

Notesnook.API/Constants.cs

@@ -14,5 +14,10 @@ namespace Notesnook.API
         public const string TagsKey = "tags";
         public const string ColorsKey = "colors";
         public const string VaultsKey = "vaults";
+        public const string InboxItemsKey = "inbox_items";
+        public const string InboxApiKeysKey = "inbox_api_keys";
+        public const string SyncDevicesKey = "sync_devices";
+        public const string DeviceIdsChunksKey = "device_ids_chunks";
+        public const string InboxItemsHistoryKey = "inbox_items_history";
     }
 }

Notesnook.API/Controllers/AnnouncementController.cs

@@ -21,10 +21,16 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using AngleSharp.Text;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using MongoDB.Driver;
+using Notesnook.API.Accessors;
 using Notesnook.API.Models;
+using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
+using Streetwriters.Common.Interfaces;
+using Streetwriters.Common.Models;
 using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Controllers
@@ -32,38 +38,45 @@ namespace Notesnook.API.Controllers
     // TODO: this should be moved out into its own microservice
     [ApiController]
     [Route("announcements")]
-    public class AnnouncementController : ControllerBase
+    public class AnnouncementController(Repository<Announcement> announcements, WampServiceAccessor serviceAccessor) : ControllerBase
     {
-        private Repository<Announcement> Announcements { get; set; }
-        public AnnouncementController(Repository<Announcement> announcements)
-        {
-            Announcements = announcements;
-        }
-
         [HttpGet("active")]
         [AllowAnonymous]
-        public async Task<IActionResult> GetActiveAnnouncements([FromQuery] string userId)
+        public async Task<IActionResult> GetActiveAnnouncements([FromQuery] string? userId)
         {
-            var totalActive = await Announcements.Collection.CountDocumentsAsync(Builders<Announcement>.Filter.Eq("IsActive", true));
-            if (totalActive <= 0) return Ok(new Announcement[] { });
-
-            var announcements = (await Announcements.FindAsync((a) => a.IsActive)).Where((a) => a.UserIds == null || a.UserIds.Length == 0 || a.UserIds.Contains(userId));
-            foreach (var announcement in announcements)
+            var filter = Builders<Announcement>.Filter.Eq(x => x.IsActive, true);
+            if (!string.IsNullOrEmpty(userId))
             {
-                if (announcement.UserIds != null && !announcement.UserIds.Contains(userId)) continue;
+                var userFilter = Builders<Announcement>.Filter.Or(
+                    Builders<Announcement>.Filter.Eq(x => x.UserIds, null),
+                    Builders<Announcement>.Filter.Size(x => x.UserIds, 0),
+                    Builders<Announcement>.Filter.AnyEq(x => x.UserIds, userId)
+                );
+                filter = Builders<Announcement>.Filter.And(filter, userFilter);
+            }
+            var userAnnouncements = await announcements.Collection.Find(filter).ToListAsync();
+            foreach (var announcement in userAnnouncements)
+            {
+                if (userId != null && announcement.UserIds != null && !announcement.UserIds.Contains(userId)) continue;
 
                 foreach (var item in announcement.Body)
                 {
                     if (item.Type != "callToActions") continue;
                     foreach (var action in item.Actions)
                     {
-                        if (action.Type != "link") continue;
+                        if (action.Type != "link" || action.Data == null) continue;
 
-                        action.Data = action.Data.Replace("{{UserId}}", userId ?? "0");
+                        action.Data = action.Data.Replace("{{UserId}}", userId ?? "");
+
+                        if (action.Data.Contains("{{Email}}"))
+                        {
+                            var user = string.IsNullOrEmpty(userId) ? null : await serviceAccessor.UserAccountService.GetUserAsync(Clients.Notesnook.Id, userId);
+                            action.Data = action.Data.Replace("{{Email}}", user?.Email ?? "");
+                        }
                     }
                 }
             }
-            return Ok(announcements);
+            return Ok(userAnnouncements);
         }
     }
 }

Notesnook.API/Controllers/InboxController.cs

@@ -0,0 +1,191 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Security.Claims;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Bson;
+using Notesnook.API.Authorization;
+using Notesnook.API.Models;
+using Notesnook.API.Services;
+using Streetwriters.Common;
+using Streetwriters.Common.Messages;
+using Streetwriters.Data.Repositories;
+
+namespace Notesnook.API.Controllers
+{
+    [ApiController]
+    [Route("inbox")]
+    public class InboxController(
+            Repository<InboxApiKey> inboxApiKeysRepository,
+            Repository<UserSettings> userSettingsRepository,
+            Repository<InboxSyncItem> inboxItemsRepository,
+            SyncDeviceService syncDeviceService,
+            ILogger<InboxController> logger) : ControllerBase
+    {
+
+        [HttpGet("api-keys")]
+        [Authorize(Policy = "Notesnook")]
+        public async Task<IActionResult> GetApiKeysAsync()
+        {
+            var userId = User.GetUserId();
+            try
+            {
+                var apiKeys = await inboxApiKeysRepository.FindAsync(t => t.UserId == userId);
+                return Ok(apiKeys);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Couldn't get inbox api keys for user {UserId}", userId);
+                return BadRequest(new { error = ex.Message });
+            }
+        }
+
+        [HttpPost("api-keys")]
+        [Authorize(Policy = "Notesnook")]
+        public async Task<IActionResult> CreateApiKeyAsync([FromBody] CreateInboxApiKeyRequest request)
+        {
+            var userId = User.GetUserId();
+            try
+            {
+                if (string.IsNullOrWhiteSpace(request.Name))
+                {
+                    return BadRequest(new { error = "Api key name is required." });
+                }
+                if (request.ExpiryDate == null)
+                {
+                    return BadRequest(new { error = "Expiry date is required." });
+                }
+
+                var count = await inboxApiKeysRepository.CountAsync(t => t.UserId == userId);
+                if (count >= 10)
+                {
+                    return BadRequest(new { error = "Maximum of 10 inbox api keys allowed." });
+                }
+
+                var inboxApiKey = new InboxApiKey
+                {
+                    UserId = userId,
+                    Name = request.Name,
+                    DateCreated = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                    ExpiryDate = request.ExpiryDate,
+                    LastUsedAt = 0
+                };
+                await inboxApiKeysRepository.InsertAsync(inboxApiKey);
+                return Ok(inboxApiKey);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Couldn't create inbox api key for {UserId}.", userId);
+                return BadRequest(new { error = ex.Message });
+            }
+        }
+
+        [HttpDelete("api-keys/{apiKey}")]
+        [Authorize(Policy = "Notesnook")]
+        public async Task<IActionResult> DeleteApiKeyAsync(string apiKey)
+        {
+            var userId = User.GetUserId();
+            try
+            {
+                if (string.IsNullOrWhiteSpace(apiKey))
+                {
+                    return BadRequest(new { error = "Api key is required." });
+                }
+
+                await inboxApiKeysRepository.DeleteAsync(t => t.UserId == userId && t.Key == apiKey);
+                return Ok(new { message = "Api key deleted successfully." });
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Couldn't delete inbox api key for user {UserId}", userId);
+                return BadRequest(new { error = ex.Message });
+            }
+        }
+
+        [HttpGet("public-encryption-key")]
+        [Authorize(Policy = InboxApiKeyAuthenticationDefaults.AuthenticationScheme)]
+        public async Task<IActionResult> GetPublicKeyAsync()
+        {
+            var userId = User.GetUserId();
+            try
+            {
+                var userSetting = await userSettingsRepository.FindOneAsync(u => u.UserId == userId);
+                if (string.IsNullOrWhiteSpace(userSetting?.InboxKeys?.Public))
+                {
+                    return NotFound(new { error = "Inbox public key is not configured." });
+                }
+                return Ok(new { key = userSetting.InboxKeys.Public });
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Couldn't get user's inbox's public key for user {UserId}", userId);
+                return BadRequest(new { error = ex.Message });
+            }
+        }
+
+        [HttpPost("items")]
+        [Authorize(Policy = InboxApiKeyAuthenticationDefaults.AuthenticationScheme)]
+        public async Task<IActionResult> CreateInboxItemAsync([FromBody] InboxSyncItem request)
+        {
+            var userId = User.GetUserId();
+            try
+            {
+                if (string.IsNullOrWhiteSpace(request.Cipher))
+                {
+                    return BadRequest(new { error = "Inbox item is required." });
+                }
+                if (string.IsNullOrWhiteSpace(request.Algorithm))
+                {
+                    return BadRequest(new { error = "Inbox item algorithm is required." });
+                }
+                if (request.Version <= 0)
+                {
+                    return BadRequest(new { error = "Valid inbox item version is required." });
+                }
+
+                request.UserId = userId;
+                request.ItemId = ObjectId.GenerateNewId().ToString();
+                await inboxItemsRepository.InsertAsync(request);
+                await syncDeviceService.AddIdsToAllDevicesAsync(userId, [new(request.ItemId, "inbox_item")]);
+                await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+                {
+                    OriginTokenId = null,
+                    UserId = userId,
+                    Message = new Message
+                    {
+                        Type = "triggerSync",
+                        Data = JsonSerializer.Serialize(new { reason = "Inbox items updated." })
+                    }
+                });
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Couldn't create inbox item for user {UserId}", userId);
+                return BadRequest(new { error = ex.Message });
+            }
+        }
+    }
+}

Notesnook.API/Controllers/MonographsController.cs

@@ -18,20 +18,27 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
 using System.Text.Json;
 using System.Threading.Tasks;
+using AngleSharp;
 using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using MongoDB.Bson;
 using MongoDB.Driver;
+using NanoidDotNet;
+using Notesnook.API.Extensions;
 using Notesnook.API.Models;
 using Notesnook.API.Services;
 using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
+using Streetwriters.Common.Enums;
+using Streetwriters.Common.Helpers;
+using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
-using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Controllers
@@ -39,21 +46,15 @@ namespace Notesnook.API.Controllers
     [ApiController]
     [Route("monographs")]
     [Authorize("Sync")]
-    public class MonographsController : ControllerBase
+    public class MonographsController(Repository<Monograph> monographs, IURLAnalyzer analyzer, SyncDeviceService syncDeviceService, WampServiceAccessor serviceAccessor, ILogger<MonographsController> logger) : ControllerBase
     {
         const string SVG_PIXEL = "<svg xmlns='http://www.w3.org/2000/svg' width='1' height='1'><circle r='9'/></svg>";
-        private Repository<Monograph> Monographs { get; set; }
-        private readonly IUnitOfWork unit;
         private const int MAX_DOC_SIZE = 15 * 1024 * 1024;
-        public MonographsController(Repository<Monograph> monographs, IUnitOfWork unitOfWork)
-        {
-            Monographs = monographs;
-            unit = unitOfWork;
-        }
 
         private static FilterDefinition<Monograph> CreateMonographFilter(string userId, Monograph monograph)
         {
             var userIdFilter = Builders<Monograph>.Filter.Eq("UserId", userId);
+            monograph.ItemId ??= monograph.Id;
             return ObjectId.TryParse(monograph.ItemId, out ObjectId id)
             ? Builders<Monograph>.Filter
                 .And(userIdFilter,
@@ -78,7 +79,7 @@ namespace Notesnook.API.Controllers
 
         private async Task<Monograph> FindMonographAsync(string userId, Monograph monograph)
         {
-            var result = await Monographs.Collection.FindAsync(CreateMonographFilter(userId, monograph), new FindOptions<Monograph>
+            var result = await monographs.Collection.FindAsync(CreateMonographFilter(userId, monograph), new FindOptions<Monograph>
             {
                 Limit = 1
             });
@@ -87,47 +88,60 @@ namespace Notesnook.API.Controllers
 
         private async Task<Monograph> FindMonographAsync(string itemId)
         {
-            var result = await Monographs.Collection.FindAsync(CreateMonographFilter(itemId), new FindOptions<Monograph>
+            var result = await monographs.Collection.FindAsync(CreateMonographFilter(itemId), new FindOptions<Monograph>
             {
                 Limit = 1
             });
             return await result.FirstOrDefaultAsync();
         }
 
+        private async Task<Monograph> FindMonographBySlugAsync(string slug)
+        {
+            var result = await monographs.Collection.FindAsync(
+                Builders<Monograph>.Filter.Eq("Slug", slug), new FindOptions<Monograph>
+                {
+                    Limit = 1
+                });
+            return await result.FirstOrDefaultAsync();
+        }
+
+        private async Task<string> GenerateUniqueSlugAsync(int length = 10, int maxAttempts = 5)
+        {
+            for (var i = 0; i < maxAttempts; i++)
+            {
+                var slug = Nanoid.Generate(size: length);
+                var exists = await monographs.Collection.Find(Builders<Monograph>.Filter.Eq("Slug", slug))
+                .Limit(1)
+                .AnyAsync();
+                if (!exists) return slug;
+            }
+            throw new Exception("Failed to generate unique slug");
+        }
+
         [HttpPost]
-        public async Task<IActionResult> PublishAsync([FromQuery] string deviceId, [FromBody] Monograph monograph)
+        public async Task<IActionResult> PublishAsync([FromQuery] string? deviceId, [FromBody] Monograph monograph)
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
-                if (userId == null) return Unauthorized();
+                var userId = this.User.GetUserId();
+                var jti = this.User.FindFirstValue("jti");
 
                 var existingMonograph = await FindMonographAsync(userId, monograph);
-                if (existingMonograph != null && !existingMonograph.Deleted)
-                {
-                    return base.Conflict("This monograph is already published.");
-                }
-
-                if (monograph.EncryptedContent == null)
-                    monograph.CompressedContent = monograph.Content.CompressBrotli();
-                monograph.UserId = userId;
-                monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-
-                if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
-                    return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
+                if (existingMonograph != null && !existingMonograph.Deleted) return await UpdateAsync(deviceId, monograph);
 
+                monograph = await CreateMonographAsync(monograph, userId);
                 if (existingMonograph != null)
                 {
-                    monograph.Id = existingMonograph?.Id;
+                    monograph.Id = existingMonograph.Id;
                 }
-                monograph.Deleted = false;
-                await Monographs.Collection.ReplaceOneAsync(
+
+                await monographs.Collection.ReplaceOneAsync(
                     CreateMonographFilter(userId, monograph),
                     monograph,
                     new ReplaceOptions { IsUpsert = true }
                 );
 
-                await MarkMonographForSyncAsync(monograph.ItemId ?? monograph.Id, deviceId);
+                await MarkMonographForSyncAsync(userId, monograph.ItemId ?? monograph.Id, deviceId, jti);
 
                 return Ok(new
                 {
@@ -137,18 +151,59 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception e)
             {
-                await Slogger<MonographsController>.Error(nameof(PublishAsync), e.ToString());
-                return BadRequest();
+                logger.LogError(e, "Failed to publish monograph");
+                return BadRequest(new { error = e.Message });
+            }
+        }
+
+        [HttpPost("v2")]
+        public async Task<IActionResult> PublishV2Async([FromQuery] string? deviceId, [FromBody] Monograph monograph)
+        {
+            try
+            {
+                var userId = this.User.GetUserId();
+                var jti = this.User.FindFirstValue("jti");
+
+                var existingMonograph = await FindMonographAsync(userId, monograph);
+                if (existingMonograph != null && !existingMonograph.Deleted) return await UpdateAsync(deviceId, monograph);
+
+                monograph = await CreateMonographAsync(monograph, userId);
+                monograph.Slug = await GenerateUniqueSlugAsync();
+
+                if (existingMonograph != null)
+                {
+                    monograph.Id = existingMonograph.Id;
+                }
+
+                await monographs.Collection.ReplaceOneAsync(
+                    CreateMonographFilter(userId, monograph),
+                    monograph,
+                    new ReplaceOptions { IsUpsert = true }
+                );
+
+                await MarkMonographForSyncAsync(userId, monograph.ItemId ?? monograph.Id, deviceId, jti);
+
+                return Ok(new
+                {
+                    id = monograph.ItemId,
+                    datePublished = monograph.DatePublished,
+                    publishUrl = Helpers.UrlHelper.ConstructPublishUrl(monograph)
+                });
+            }
+            catch (Exception e)
+            {
+                logger.LogError(e, "Failed to publish monograph");
+                return BadRequest(new { error = e.Message });
             }
         }
 
         [HttpPatch]
-        public async Task<IActionResult> UpdateAsync([FromQuery] string deviceId, [FromBody] Monograph monograph)
+        public async Task<IActionResult> UpdateAsync([FromQuery] string? deviceId, [FromBody] Monograph monograph)
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
-                if (userId == null) return Unauthorized();
+                var userId = this.User.GetUserId();
+                var jti = this.User.FindFirstValue("jti");
 
                 var existingMonograph = await FindMonographAsync(userId, monograph);
                 if (existingMonograph == null || existingMonograph.Deleted)
@@ -159,13 +214,17 @@ namespace Notesnook.API.Controllers
                 if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
                     return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
 
+                var sanitizationLevel = ContentSanitizationLevel.Unknown;
                 if (monograph.EncryptedContent == null)
-                    monograph.CompressedContent = monograph.Content.CompressBrotli();
+                {
+                    sanitizationLevel = User.IsUserSubscribed() ? ContentSanitizationLevel.Partial : ContentSanitizationLevel.Full;
+                    monograph.CompressedContent = (await SanitizeContentAsync(monograph.Content, sanitizationLevel)).CompressBrotli();
+                }
                 else
                     monograph.Content = null;
 
                 monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                var result = await Monographs.Collection.UpdateOneAsync(
+                var result = await monographs.Collection.UpdateOneAsync(
                     CreateMonographFilter(userId, monograph),
                     Builders<Monograph>.Update
                     .Set(m => m.DatePublished, monograph.DatePublished)
@@ -174,40 +233,41 @@ namespace Notesnook.API.Controllers
                     .Set(m => m.SelfDestruct, monograph.SelfDestruct)
                     .Set(m => m.Title, monograph.Title)
                     .Set(m => m.Password, monograph.Password)
+                    .Set(m => m.ContentSanitizationLevel, sanitizationLevel)
                 );
                 if (!result.IsAcknowledged) return BadRequest();
 
-                await MarkMonographForSyncAsync(monograph.ItemId ?? monograph.Id, deviceId);
+                await MarkMonographForSyncAsync(userId, monograph.ItemId ?? monograph.Id, deviceId, jti);
 
                 return Ok(new
                 {
                     id = monograph.ItemId,
                     datePublished = monograph.DatePublished,
+                    publishUrl = Helpers.UrlHelper.ConstructPublishUrl(existingMonograph)
                 });
             }
             catch (Exception e)
             {
-                await Slogger<MonographsController>.Error(nameof(UpdateAsync), e.ToString());
-                return BadRequest();
+                logger.LogError(e, "Failed to update monograph");
+                return BadRequest(new { error = e.Message });
             }
         }
 
         [HttpGet]
         public async Task<IActionResult> GetUserMonographsAsync()
         {
-            var userId = this.User.FindFirstValue("sub");
-            if (userId == null) return Unauthorized();
+            var userId = this.User.GetUserId();
 
-            var monographs = (await Monographs.Collection.FindAsync(
+            var userMonographs = (await monographs.Collection.FindAsync(
                     Builders<Monograph>.Filter.And(
                         Builders<Monograph>.Filter.Eq("UserId", userId),
-                        Builders<Monograph>.Filter.Eq("Deleted", false)
+                        Builders<Monograph>.Filter.Ne("Deleted", true)
                     )
                , new FindOptions<Monograph, ObjectWithId>
                {
                    Projection = Builders<Monograph>.Projection.Include("_id").Include("ItemId"),
                })).ToEnumerable();
-            return Ok(monographs.Select((m) => m.ItemId ?? m.Id));
+            return Ok(userMonographs.Select((m) => m.ItemId ?? m.Id));
         }
 
         [HttpGet("{id}")]
@@ -224,10 +284,7 @@ namespace Notesnook.API.Controllers
                 });
             }
 
-            if (monograph.EncryptedContent == null)
-                monograph.Content = monograph.CompressedContent.DecompressBrotli();
-            if (monograph.ItemId == null) monograph.ItemId = monograph.Id;
-            return Ok(monograph);
+            return Ok(await ProcessMonographAsync(monograph));
         }
 
         [HttpGet("{id}/view")]
@@ -235,31 +292,33 @@ namespace Notesnook.API.Controllers
         public async Task<IActionResult> TrackView([FromRoute] string id)
         {
             var monograph = await FindMonographAsync(id);
-            if (monograph == null || monograph.Deleted) return Content(SVG_PIXEL, "image/svg+xml");
+            if (monograph == null || monograph.Deleted)
+                return Content(SVG_PIXEL, "image/svg+xml");
 
-            if (monograph.SelfDestruct)
-            {
-                var userId = this.User.FindFirstValue("sub");
-                await Monographs.Collection.ReplaceOneAsync(
-                    CreateMonographFilter(userId, monograph),
-                    new Monograph
-                    {
-                        ItemId = id,
-                        Id = monograph.Id,
-                        Deleted = true
-                    }
-                );
-
-                await MarkMonographForSyncAsync(id);
-            }
+            var cookieName = $"viewed_{id}";
+            await TrackViewAsync(monograph, cookieName, $"/monographs/{id}");
 
             return Content(SVG_PIXEL, "image/svg+xml");
         }
 
-        [HttpDelete("{id}")]
-        public async Task<IActionResult> DeleteAsync([FromQuery] string deviceId, [FromRoute] string id)
+        [HttpGet("v2/{slug}/view")]
+        [AllowAnonymous]
+        public async Task<IActionResult> TrackViewV2([FromRoute] string slug)
         {
-            var monograph = await FindMonographAsync(id);
+            var monograph = await FindMonographBySlugAsync(slug);
+            if (monograph == null || monograph.Deleted)
+                return Content(SVG_PIXEL, "image/svg+xml");
+
+            var cookieName = $"viewed_{slug}";
+            await TrackViewAsync(monograph, cookieName, $"/monographs/v2/{slug}");
+            return Content(SVG_PIXEL, "image/svg+xml");
+        }
+
+        [HttpGet("v2/{slug}")]
+        [AllowAnonymous]
+        public async Task<IActionResult> GetMonographBySlugAsync([FromRoute] string slug)
+        {
+            var monograph = await FindMonographBySlugAsync(slug);
             if (monograph == null || monograph.Deleted)
             {
                 return NotFound(new
@@ -269,55 +328,239 @@ namespace Notesnook.API.Controllers
                 });
             }
 
-            var userId = this.User.FindFirstValue("sub");
-            await Monographs.Collection.ReplaceOneAsync(
+            return Ok(await ProcessMonographAsync(monograph));
+        }
+
+        [HttpGet("{id}/analytics")]
+        [Obsolete("This endpoint is deprecated and will be removed in future versions. Use GET /monographs/{id}/metadata instead.")]
+        public async Task<IActionResult> GetMonographAnalyticsAsync([FromRoute] string id)
+        {
+            if (!FeatureAuthorizationHelper.IsFeatureAllowed(Features.MONOGRAPH_ANALYTICS, Clients.Notesnook.Id, User))
+                return BadRequest(new { error = "Monograph analytics are only available on the Pro & Believer plans." });
+
+            var userId = this.User.GetUserId();
+            var monograph = await FindMonographAsync(id);
+            if (monograph == null || monograph.Deleted || monograph.UserId != userId)
+            {
+                return NotFound();
+            }
+
+            return Ok(new { totalViews = monograph.ViewCount });
+        }
+
+        [HttpDelete("{id}")]
+        public async Task<IActionResult> DeleteAsync([FromQuery] string? deviceId, [FromRoute] string id)
+        {
+            var userId = this.User.GetUserId();
+
+            var monograph = await FindMonographAsync(id);
+            if (monograph == null || monograph.Deleted)
+                return Ok();
+
+            var jti = this.User.FindFirstValue("jti");
+
+            await monographs.Collection.ReplaceOneAsync(
                 CreateMonographFilter(userId, monograph),
                 new Monograph
                 {
                     ItemId = id,
                     Id = monograph.Id,
                     Deleted = true,
-                    UserId = monograph.UserId
+                    UserId = monograph.UserId,
+                    ViewCount = 0
                 }
             );
 
-            await MarkMonographForSyncAsync(id, deviceId);
+            await MarkMonographForSyncAsync(userId, id, deviceId, jti);
 
             return Ok();
         }
 
-        private async Task MarkMonographForSyncAsync(string monographId, string deviceId)
+        [HttpGet("{id}/metadata")]
+        public async Task<IActionResult> GetMetadataAsync([FromRoute] string id)
         {
-            if (deviceId == null) return;
-            var userId = this.User.FindFirstValue("sub");
-
-            new SyncDeviceService(new SyncDevice(userId, deviceId)).AddIdsToOtherDevices([$"{monographId}:monograph"]);
-            await SendTriggerSyncEventAsync();
-        }
-
-        private async Task MarkMonographForSyncAsync(string monographId)
-        {
-            var userId = this.User.FindFirstValue("sub");
-
-            new SyncDeviceService(new SyncDevice(userId, string.Empty)).AddIdsToAllDevices([$"{monographId}:monograph"]);
-            await SendTriggerSyncEventAsync(sendToAllDevices: true);
-        }
-
-        private async Task SendTriggerSyncEventAsync(bool sendToAllDevices = false)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            var jti = this.User.FindFirstValue("jti");
-
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+            var userId = this.User.GetUserId();
+            var monograph = await FindMonographAsync(id);
+            if (monograph == null || monograph.Deleted || monograph.UserId != userId)
             {
-                OriginTokenId = sendToAllDevices ? null : jti,
-                UserId = userId,
-                Message = new Message
+                return NotFound();
+            }
+
+            var isPro = FeatureAuthorizationHelper.IsFeatureAllowed(Features.MONOGRAPH_ANALYTICS, Clients.Notesnook.Id, User);
+            var totalViews = isPro ? monograph.ViewCount : 0;
+
+            return Ok(new
+            {
+                publishUrl = Helpers.UrlHelper.ConstructPublishUrl(monograph),
+                analytics = new
                 {
-                    Type = "triggerSync",
-                    Data = JsonSerializer.Serialize(new { reason = "Monographs updated." })
+                    totalViews
                 }
             });
         }
+
+        private async Task MarkMonographForSyncAsync(string userId, string monographId, string? deviceId, string? jti)
+        {
+            if (deviceId == null) return;
+
+            await syncDeviceService.AddIdsToOtherDevicesAsync(userId, deviceId, [new(monographId, "monograph")]);
+        }
+
+        private async Task MarkMonographForSyncAsync(string userId, string monographId)
+        {
+            await syncDeviceService.AddIdsToAllDevicesAsync(userId, [new(monographId, "monograph")]);
+        }
+
+        // (selector, url-bearing attribute) pairs to inspect
+        private static readonly (string Selector, string Attribute)[] urlElements =
+        [
+            ("a", "href"),
+            ("img", "src"),
+            ("iframe", "src"),
+            ("embed", "src"),
+            ("object", "data"),
+            ("source", "src"),
+            ("video", "src"),
+            ("audio", "src"),
+        ];
+
+        private async Task<Monograph> CreateMonographAsync(Monograph monograph, string userId)
+        {
+            if (monograph.EncryptedContent == null)
+            {
+                var sanitizationLevel = User.IsUserSubscribed() ? ContentSanitizationLevel.Partial : ContentSanitizationLevel.Full;
+                monograph.CompressedContent = (await SanitizeContentAsync(monograph.Content, sanitizationLevel)).CompressBrotli();
+                monograph.ContentSanitizationLevel = sanitizationLevel;
+            }
+
+            monograph.UserId = userId;
+            monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+
+            if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
+                throw new Exception("Monograph is too big. Max allowed size is 15mb.");
+
+            monograph.Deleted = false;
+            monograph.ViewCount = 0;
+
+            return monograph;
+        }
+
+        private async Task TrackViewAsync(Monograph monograph, string cookieName, string cookiePath)
+        {
+            var hasVisitedBefore = Request.Cookies.ContainsKey(cookieName);
+
+            if (monograph.SelfDestruct)
+            {
+                await monographs.Collection.ReplaceOneAsync(
+                    CreateMonographFilter(monograph.UserId!, monograph),
+                    new Monograph
+                    {
+                        ItemId = monograph.ItemId,
+                        Id = monograph.Id,
+                        Deleted = true,
+                        UserId = monograph.UserId,
+                        ViewCount = 0
+                    }
+                );
+                await MarkMonographForSyncAsync(monograph.UserId!, monograph.ItemId ?? monograph.Id);
+            }
+            else if (!hasVisitedBefore)
+            {
+                await monographs.Collection.UpdateOneAsync(
+                    CreateMonographFilter(monograph.UserId!, monograph),
+                    Builders<Monograph>.Update.Inc(m => m.ViewCount, 1)
+                );
+
+                var cookieOptions = new CookieOptions
+                {
+                    Path = cookiePath,
+                    HttpOnly = true,
+                    Secure = Request.IsHttps,
+                    Expires = DateTimeOffset.UtcNow.AddMonths(1)
+                };
+                Response.Cookies.Append(cookieName, "1", cookieOptions);
+            }
+        }
+
+        private async Task<Monograph> ProcessMonographAsync(Monograph monograph)
+        {
+
+            if (monograph.EncryptedContent == null)
+            {
+                var isContentUnsanitized = monograph.ContentSanitizationLevel == ContentSanitizationLevel.Partial || monograph.ContentSanitizationLevel == ContentSanitizationLevel.Unknown;
+                if (!Constants.IS_SELF_HOSTED && isContentUnsanitized && serviceAccessor.UserSubscriptionService != null && !await serviceAccessor.UserSubscriptionService.IsUserSubscribedAsync(Clients.Notesnook.Id, monograph.UserId!))
+                {
+                    var cleaned = await SanitizeContentAsync(monograph.CompressedContent?.DecompressBrotli(), ContentSanitizationLevel.Full);
+                    monograph.CompressedContent = cleaned.CompressBrotli();
+                    await monographs.Collection.UpdateOneAsync(
+                        CreateMonographFilter(monograph.UserId!, monograph),
+                        Builders<Monograph>.Update
+                            .Set(m => m.CompressedContent, monograph.CompressedContent)
+                            .Set(m => m.ContentSanitizationLevel, ContentSanitizationLevel.Full)
+                    );
+                }
+                monograph.Content = monograph.CompressedContent?.DecompressBrotli();
+            }
+
+            monograph.ItemId ??= monograph.Id;
+            return monograph;
+        }
+
+        private async Task<string> SanitizeContentAsync(string? content, ContentSanitizationLevel level)
+        {
+            if (string.IsNullOrEmpty(content)) return string.Empty;
+            if (Constants.IS_SELF_HOSTED) return content;
+            try
+            {
+                var json = JsonSerializer.Deserialize<MonographContent>(content) ?? throw new Exception("Invalid monograph content.");
+                var html = json.Data;
+
+                if (level == ContentSanitizationLevel.Partial)
+                {
+                    var config = Configuration.Default.WithDefaultLoader();
+                    var context = BrowsingContext.New(config);
+                    var document = await context.OpenAsync(r => r.Content(html));
+
+                    foreach (var (selector, attribute) in urlElements)
+                    {
+                        foreach (var element in document.QuerySelectorAll(selector))
+                        {
+                            var url = element.GetAttribute(attribute);
+                            if (string.IsNullOrEmpty(url)) continue;
+                            if (!await analyzer.IsURLSafeAsync(url))
+                            {
+                                logger.LogInformation("Malicious URL detected in <{Selector} {Attribute}>: {Url}", selector, attribute, url);
+                                element.RemoveAttribute(attribute);
+                            }
+                        }
+                    }
+
+                    html = document.ToHtml();
+                }
+                else if (level == ContentSanitizationLevel.Full)
+                {
+                    var config = Configuration.Default.WithDefaultLoader();
+                    var context = BrowsingContext.New(config);
+                    var document = await context.OpenAsync(r => r.Content(html));
+                    foreach (var element in document.QuerySelectorAll("a,iframe,img,object,svg,button,link"))
+                    {
+                        foreach (var attr in element.Attributes.ToList())
+                            element.RemoveAttribute(attr.Name);
+                    }
+                    html = document.ToHtml();
+                }
+
+                return JsonSerializer.Serialize<MonographContent>(new MonographContent
+                {
+                    Type = json.Type,
+                    Data = html
+                });
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Failed to cleanup monograph content");
+                return content;
+            }
+        }
     }
 }

Notesnook.API/Controllers/S3Controller.cs

@@ -17,112 +17,218 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Mvc;
-using Amazon.S3.Model;
-using System.Threading.Tasks;
-using System.Security.Claims;
-using Notesnook.API.Interfaces;
 using System;
+using System.Net.Http;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Http.Extensions;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Notesnook.API.Helpers;
+using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
+using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
+using Streetwriters.Common.Extensions;
+using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Route("s3")]
     [ResponseCache(NoStore = true, Location = ResponseCacheLocation.None)]
-    public class S3Controller : ControllerBase
+    [Authorize("Sync")]
+    public class S3Controller(IS3Service s3Service, ISyncItemsRepositoryAccessor repositories, WampServiceAccessor serviceAccessor, ILogger<S3Controller> logger) : ControllerBase
     {
-        private IS3Service S3Service { get; set; }
-        public S3Controller(IS3Service s3Service)
-        {
-            S3Service = s3Service;
-        }
-
         [HttpPut]
-        [Authorize("Pro")]
-        public IActionResult Upload([FromQuery] string name)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            var url = S3Service.GetUploadObjectUrl(userId, name);
-            if (url == null) return BadRequest("Could not create signed url.");
-            return Ok(url);
-        }
-
-
-        [HttpGet("multipart")]
-        [Authorize("Pro")]
-        public async Task<IActionResult> MultipartUpload([FromQuery] string name, [FromQuery] int parts, [FromQuery] string uploadId)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            try
-            {
-                var meta = await S3Service.StartMultipartUploadAsync(userId, name, parts, uploadId);
-                return Ok(meta);
-            }
-            catch (Exception ex) { return BadRequest(ex.Message); }
-        }
-
-        [HttpDelete("multipart")]
-        [Authorize("Pro")]
-        public async Task<IActionResult> AbortMultipartUpload([FromQuery] string name, [FromQuery] string uploadId)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            try
-            {
-                await S3Service.AbortMultipartUploadAsync(userId, name, uploadId);
-                return Ok();
-            }
-            catch (Exception ex) { return BadRequest(ex.Message); }
-        }
-
-        [HttpPost("multipart")]
-        [Authorize("Pro")]
-        public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequestWrapper uploadRequestWrapper)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            try
-            {
-                await S3Service.CompleteMultipartUploadAsync(userId, uploadRequestWrapper.ToRequest());
-                return Ok();
-            }
-            catch (Exception ex) { return BadRequest(ex.Message); }
-        }
-
-        [HttpGet]
-        [Authorize("Sync")]
-        public IActionResult Download([FromQuery] string name)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            var url = S3Service.GetDownloadObjectUrl(userId, name);
-            if (url == null) return BadRequest("Could not create signed url.");
-            return Ok(url);
-        }
-
-        [HttpHead]
-        [Authorize("Sync")]
-        public async Task<IActionResult> Info([FromQuery] string name)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            var size = await S3Service.GetObjectSizeAsync(userId, name);
-            HttpContext.Response.Headers.ContentLength = size;
-            return Ok();
-        }
-
-        [HttpDelete]
-        [Authorize("Sync")]
-        public async Task<IActionResult> DeleteAsync([FromQuery] string name)
+        public async Task<IActionResult> Upload([FromQuery] string name)
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub");
-                await S3Service.DeleteObjectAsync(userId, name);
+                var userId = this.User.GetUserId();
+
+                var fileSize = HttpContext.Request.ContentLength ?? 0;
+                bool hasBody = fileSize > 0;
+
+                if (!hasBody)
+                {
+                    return Ok(Request.GetEncodedUrl() + "&access_token=" + Request.Headers.Authorization.ToString().Replace("Bearer ", ""));
+                }
+
+                if (Constants.IS_SELF_HOSTED) await UploadFileAsync(userId, name, fileSize);
+                else await UploadFileWithChecksAsync(userId, name, fileSize);
+
                 return Ok();
             }
             catch (Exception ex)
             {
-                return BadRequest(ex.Message);
+                logger.LogError(ex, "Error uploading attachment for user.");
+                return BadRequest(new { error = "Failed to upload attachment." });
             }
         }
+
+        private async Task UploadFileWithChecksAsync(string userId, string name, long fileSize)
+        {
+            var userSettings = await repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+
+            var subscription = await serviceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
+
+            if (StorageHelper.IsFileSizeExceeded(subscription, fileSize))
+                throw new Exception("Max file size exceeded.");
+
+            userSettings.StorageLimit = StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+            if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value + fileSize))
+                throw new Exception("Storage limit exceeded.");
+
+            var uploadedFileSize = await UploadFileAsync(userId, name, fileSize);
+
+            userSettings.StorageLimit.Value += uploadedFileSize;
+            await repositories.UsersSettings.Collection.UpdateOneAsync(
+                Builders<UserSettings>.Filter.Eq(u => u.UserId, userId),
+                Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
+            );
+
+            // extra check in case user sets wrong ContentLength in the HTTP header
+            if (uploadedFileSize != fileSize && StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value))
+            {
+                await s3Service.DeleteObjectAsync(userId, name);
+                throw new Exception("Storage limit exceeded.");
+            }
+        }
+
+        private async Task<long> UploadFileAsync(string userId, string name, long fileSize)
+        {
+            var url = await s3Service.GetInternalUploadObjectUrlAsync(userId, name) ?? throw new Exception("Could not create signed url.");
+
+            var httpClient = new HttpClient();
+            var content = new StreamContent(HttpContext.Request.BodyReader.AsStream());
+            content.Headers.ContentLength = fileSize;
+            var response = await httpClient.SendRequestAsync<Response>(url, null, HttpMethod.Put, content);
+            if (!response.Success) throw new Exception(response.Content != null ? await response.Content.ReadAsStringAsync() : "Could not upload file.");
+
+            return await s3Service.GetObjectSizeAsync(userId, name);
+        }
+
+
+        [HttpGet("multipart")]
+        public async Task<IActionResult> MultipartUpload([FromQuery] string name, [FromQuery] int parts, [FromQuery] string? uploadId)
+        {
+            var userId = this.User.GetUserId();
+            try
+            {
+                var meta = await s3Service.StartMultipartUploadAsync(userId, name, parts, uploadId);
+                return Ok(meta);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error starting multipart upload for user.");
+                return BadRequest(new { error = "Failed to start multipart upload." });
+            }
+        }
+
+        [HttpDelete("multipart")]
+        public async Task<IActionResult> AbortMultipartUpload([FromQuery] string name, [FromQuery] string uploadId)
+        {
+            var userId = this.User.GetUserId();
+            try
+            {
+                await s3Service.AbortMultipartUploadAsync(userId, name, uploadId);
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error aborting multipart upload for user.");
+                return BadRequest(new { error = "Failed to abort multipart upload." });
+            }
+        }
+
+        [HttpPost("multipart")]
+        public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequestWrapper uploadRequestWrapper)
+        {
+            var userId = this.User.GetUserId();
+            try
+            {
+                await s3Service.CompleteMultipartUploadAsync(userId, uploadRequestWrapper.ToRequest());
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error completing multipart upload for user.");
+                return BadRequest(new { error = "Failed to complete multipart upload." });
+            }
+        }
+
+        [HttpGet]
+        public async Task<IActionResult> Download([FromQuery] string name)
+        {
+            try
+            {
+                var userId = this.User.GetUserId();
+                var url = await s3Service.GetDownloadObjectUrlAsync(userId, name);
+                if (url == null) return BadRequest("Could not create signed url.");
+                return Ok(url);
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error generating download url for user.");
+                return BadRequest(new { error = "Failed to get attachment url." });
+            }
+        }
+
+        [HttpHead]
+        public async Task<IActionResult> Info([FromQuery] string name)
+        {
+            try
+            {
+                var userId = this.User.GetUserId();
+                var size = await s3Service.GetObjectSizeAsync(userId, name); Response.Headers.ContentLength = size;
+                Response.Headers["X-Object-Size"] = size.ToString();
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error getting object info for user.");
+                return BadRequest(new { error = "Failed to get attachment info." });
+            }
+        }
+
+        [HttpDelete]
+        public async Task<IActionResult> DeleteAsync([FromQuery] string name)
+        {
+            try
+            {
+                var userId = this.User.GetUserId();
+                await s3Service.DeleteObjectAsync(userId, name);
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error deleting object for user.");
+                return BadRequest(new { error = "Failed to delete attachment." });
+            }
+        }
+
+        // [HttpPost("bulk-delete")]
+        // public async Task<IActionResult> DeleteBulkAsync([FromBody] DeleteBulkObjectsRequest request)
+        // {
+        //     try
+        //     {
+        //         if (request.Names == null || request.Names.Length == 0)
+        //         {
+        //             return BadRequest(new { error = "No files specified for deletion." });
+        //         }
+
+        //         var userId = this.User.GetUserId();
+        //         await s3Service.DeleteObjectsAsync(userId, request.Names);
+        //         return Ok();
+        //     }
+        //     catch (Exception ex)
+        //     {
+        //         logger.LogError(ex, "Error deleting objects for user.");
+        //         return BadRequest(new { error = "Failed to delete attachments." });
+        //     }
+        // }
     }
-}
+}

Notesnook.API/Controllers/SyncDeviceController.cs

@@ -24,6 +24,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models.Responses;
 using Notesnook.API.Services;
@@ -36,20 +37,20 @@ namespace Notesnook.API.Controllers
     [ApiController]
     [Authorize]
     [Route("devices")]
-    public class SyncDeviceController : ControllerBase
+    public class SyncDeviceController(SyncDeviceService syncDeviceService, ILogger<SyncDeviceController> logger) : ControllerBase
     {
         [HttpPost]
         public async Task<IActionResult> RegisterDevice([FromQuery] string deviceId)
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub") ?? throw new Exception("User not found.");
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).RegisterDevice();
+                var userId = this.User.GetUserId();
+                await syncDeviceService.RegisterDeviceAsync(userId, deviceId);
                 return Ok();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't register device.", ex.ToString());
+                logger.LogError(ex, "Failed to register device: {DeviceId}", deviceId);
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -60,13 +61,13 @@ namespace Notesnook.API.Controllers
         {
             try
             {
-                var userId = this.User.FindFirstValue("sub") ?? throw new Exception("User not found.");
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).UnregisterDevice();
+                var userId = this.User.GetUserId();
+                await syncDeviceService.UnregisterDeviceAsync(userId, deviceId);
                 return Ok();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't unregister device.", ex.ToString());
+                logger.LogError(ex, "Failed to unregister device: {DeviceId}", deviceId);
                 return BadRequest(new { error = ex.Message });
             }
         }

Notesnook.API/Controllers/UsersController.cs

@@ -18,35 +18,41 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
+using System.Net.Http;
 using System.Security.Claims;
+using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http.Timeouts;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
 using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
+using Streetwriters.Common.Extensions;
+using Streetwriters.Common.Messages;
+using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Authorize]
     [Route("users")]
-    public class UsersController(IUserService UserService) : ControllerBase
+    public class UsersController(IUserService UserService, WampServiceAccessor serviceAccessor, ILogger<UsersController> logger) : ControllerBase
     {
         [HttpPost]
         [AllowAnonymous]
-        public async Task<IActionResult> Signup()
+        public async Task<IActionResult> Signup([FromForm] SignupForm form)
         {
             try
             {
-                await UserService.CreateUserAsync();
-                return Ok();
+                return Ok(await UserService.CreateUserAsync(form));
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(Signup), "Couldn't sign up.", ex.ToString());
+                logger.LogError(ex, "Failed to sign up user");
                 return BadRequest(new { error = ex.Message });
             }
         }
@@ -54,45 +60,78 @@ namespace Notesnook.API.Controllers
         [HttpGet]
         public async Task<IActionResult> GetUser()
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
                 UserResponse response = await UserService.GetUserAsync(userId);
-                if (!response.Success) return BadRequest(response);
+                if (!response.Success) return BadRequest();
                 return Ok(response);
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't get user for id.", userId, ex.ToString());
+                logger.LogError(ex, "Failed to get user with id: {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
 
         [HttpPatch]
-        public async Task<IActionResult> UpdateUser([FromBody] UserResponse user)
+        public async Task<IActionResult> UpdateUser([FromBody] UserKeys keys)
         {
-            var userId = User.FindFirstValue("sub");
+            var userId = User.GetUserId();
             try
             {
-                var keys = new UserKeys
-                {
-                    AttachmentsKey = user.AttachmentsKey,
-                    MonographPasswordsKey = user.MonographPasswordsKey
-                };
                 await UserService.SetUserKeysAsync(userId, keys);
                 return Ok();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't update user with id.", userId, ex.ToString());
+                logger.LogError(ex, "Failed to update user with id: {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }
 
+        [HttpPatch("password/{type}")]
+        public async Task<IActionResult> ChangePassword([FromRoute] string type, [FromBody] ChangePasswordForm form)
+        {
+            return BadRequest(new { error = "Password change is currently disabled." });
+            // var userId = User.GetUserId();
+            // var clientId = User.FindFirstValue("client_id");
+            // var jti = User.FindFirstValue("jti");
+            // var isPasswordReset = type == "reset";
+            // try
+            // {
+            //     var result = isPasswordReset ? await serviceAccessor.UserAccountService.ResetPasswordAsync(userId, form.NewPassword) : await serviceAccessor.UserAccountService.ChangePasswordAsync(userId, form.OldPassword, form.NewPassword);
+            //     if (!result)
+            //         return BadRequest("Failed to change password.");
+
+            //     await UserService.SetUserKeysAsync(userId, form.UserKeys);
+
+            //     await serviceAccessor.UserAccountService.ClearSessionsAsync(userId, clientId, all: false, jti, null);
+
+            //     await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+            //     {
+            //         UserId = userId,
+            //         OriginTokenId = jti,
+            //         Message = new Message
+            //         {
+            //             Type = "logout",
+            //             Data = JsonSerializer.Serialize(new { reason = "Password changed." })
+            //         }
+            //     });
+
+            //     return Ok();
+            // }
+            // catch (Exception ex)
+            // {
+            //     logger.LogError(ex, "Failed to change password");
+            //     return BadRequest(new { error = ex.Message });
+            // }
+        }
+
         [HttpPost("reset")]
         public async Task<IActionResult> Reset([FromForm] bool removeAttachments)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
 
             if (await UserService.ResetUserAsync(userId, removeAttachments))
                 return Ok();
@@ -103,7 +142,7 @@ namespace Notesnook.API.Controllers
         [RequestTimeout(5 * 60 * 1000)]
         public async Task<IActionResult> Delete([FromForm] DeleteAccountForm form)
         {
-            var userId = this.User.FindFirstValue("sub");
+            var userId = this.User.GetUserId();
             var jti = User.FindFirstValue("jti");
             try
             {
@@ -112,7 +151,7 @@ namespace Notesnook.API.Controllers
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't delete user with id.", userId, ex.ToString());
+                logger.LogError(ex, "Failed to delete user with id: {UserId}", userId);
                 return BadRequest(new { error = ex.Message });
             }
         }

Notesnook.API/Dockerfile

@@ -1,7 +1,7 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/runtime-deps:9.0-alpine AS base
 WORKDIR /app
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
+FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:9.0-alpine AS build
 ARG TARGETARCH
 ARG BUILDPLATFORM
 ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
@@ -27,8 +27,6 @@ FROM build AS publish
 RUN dotnet publish -c Release -o /app/publish \
   #--runtime alpine-x64 \
   --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
   /p:PublishSingleFile=true \
   /p:JsonSerializerIsReflectionEnabledByDefault=true \
   -a $TARGETARCH

Notesnook.API/EventSources/SyncEventCounterSource.cs

@@ -7,19 +7,17 @@ public sealed class SyncEventCounterSource : EventSource
 {
     public static readonly SyncEventCounterSource Log = new();
 
-    private Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
-    private Counter<int> fetchCounter;
-    private Counter<int> pushCounter;
-    private Counter<int> legacyFetchCounter;
-    private Counter<int> pushV2Counter;
-    private Counter<int> fetchV2Counter;
-    private Histogram<long> fetchV2Duration;
-    private Histogram<long> pushV2Duration;
+    private readonly Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
+    private readonly Counter<int> fetchCounter;
+    private readonly Counter<int> pushCounter;
+    private readonly Counter<int> pushV2Counter;
+    private readonly Counter<int> fetchV2Counter;
+    private readonly Histogram<long> fetchV2Duration;
+    private readonly Histogram<long> pushV2Duration;
     private SyncEventCounterSource()
     {
         fetchCounter = meter.CreateCounter<int>("sync.fetches", "fetches", "Total fetches");
         pushCounter = meter.CreateCounter<int>("sync.pushes", "pushes", "Total pushes");
-        legacyFetchCounter = meter.CreateCounter<int>("sync.legacy-fetches", "fetches", "Total legacy fetches");
         fetchV2Counter = meter.CreateCounter<int>("sync.v2.fetches", "fetches", "Total v2 fetches");
         pushV2Counter = meter.CreateCounter<int>("sync.v2.pushes", "pushes", "Total v2 pushes");
         fetchV2Duration = meter.CreateHistogram<long>("sync.v2.fetch_duration");
@@ -27,7 +25,6 @@ public sealed class SyncEventCounterSource : EventSource
     }
 
     public void Fetch() => fetchCounter.Add(1);
-    public void LegacyFetch() => legacyFetchCounter.Add(1);
     public void FetchV2() => fetchV2Counter.Add(1);
     public void PushV2() => pushV2Counter.Add(1);
     public void Push() => pushCounter.Add(1);
@@ -36,14 +33,7 @@ public sealed class SyncEventCounterSource : EventSource
 
     protected override void Dispose(bool disposing)
     {
-        legacyFetchCounter = null;
-        fetchV2Counter = null;
-        pushV2Counter = null;
-        pushCounter = null;
-        fetchCounter = null;
         meter.Dispose();
-        meter = null;
-
         base.Dispose(disposing);
     }
 }

Notesnook.API/Extensions/AuthorizationResultTransformer.cs

@@ -17,8 +17,11 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using System.Linq;
 using System.Net;
+using System.Net.WebSockets;
+using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
@@ -42,12 +45,9 @@ namespace Notesnook.API.Extensions
             AuthorizationPolicy authorizationPolicy,
             PolicyAuthorizationResult policyAuthorizationResult)
         {
-            var isWebsocket = httpContext.Request.Headers.Upgrade == "websocket";
-
-            if (!isWebsocket && policyAuthorizationResult.Forbidden && policyAuthorizationResult.AuthorizationFailure != null)
+            if (policyAuthorizationResult.Forbidden && policyAuthorizationResult.AuthorizationFailure != null)
             {
                 var error = string.Join("\n", policyAuthorizationResult.AuthorizationFailure.FailureReasons.Select((r) => r.Message));
-
                 if (!string.IsNullOrEmpty(error))
                 {
                     httpContext.Response.StatusCode = (int)HttpStatusCode.Unauthorized;
@@ -55,17 +55,8 @@ namespace Notesnook.API.Extensions
                     await httpContext.Response.WriteAsync(JsonSerializer.Serialize(new { error }));
                     return;
                 }
-
-                await _handler.HandleAsync(requestDelegate, httpContext, authorizationPolicy, policyAuthorizationResult);
-            }
-            else if (isWebsocket)
-            {
-                await _handler.HandleAsync(requestDelegate, httpContext, authorizationPolicy, PolicyAuthorizationResult.Success());
-            }
-            else
-            {
-                await _handler.HandleAsync(requestDelegate, httpContext, authorizationPolicy, policyAuthorizationResult);
             }
+            await _handler.HandleAsync(requestDelegate, httpContext, authorizationPolicy, policyAuthorizationResult);
         }
     }
 }

Notesnook.API/Extensions/ClaimsPrincipalExtensions.cs

@@ -0,0 +1,17 @@
+using System.Threading;
+using System;
+using System.Threading.Tasks;
+using System.Linq;
+
+namespace System.Security.Claims
+{
+    public static class ClaimsPrincipalExtensions
+    {
+        private readonly static string[] SUBSCRIBED_CLAIMS = ["believer", "education", "essential", "pro", "legacy_pro"];
+        public static bool IsUserSubscribed(this ClaimsPrincipal user)
+         => user.Claims.Any((c) => c.Type == "notesnook:status" && SUBSCRIBED_CLAIMS.Contains(c.Value));
+
+        public static string GetUserId(this ClaimsPrincipal user)
+        => user.FindFirstValue("sub") ?? throw new Exception("User ID not found in claims.");
+    }
+}

Notesnook.API/Helpers/S3FailoverHelper.cs

@@ -0,0 +1,414 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Microsoft.Extensions.Logging;
+
+namespace Notesnook.API.Helpers
+{
+    /// <summary>
+    /// Configuration for S3 failover behavior
+    /// </summary>
+    public class S3FailoverConfig
+    {
+        /// <summary>
+        /// Maximum number of retry attempts per endpoint
+        /// </summary>
+        public int MaxRetries { get; set; } = 3;
+
+        /// <summary>
+        /// Delay between retries in milliseconds
+        /// </summary>
+        public int RetryDelayMs { get; set; } = 1000;
+
+        /// <summary>
+        /// Whether to use exponential backoff for retries
+        /// </summary>
+        public bool UseExponentialBackoff { get; set; } = true;
+
+        /// <summary>
+        /// Whether to allow failover for write operations (PUT, POST, DELETE).
+        /// Default is false to prevent data consistency issues.
+        /// </summary>
+        public bool AllowWriteFailover { get; set; } = false;
+
+        /// <summary>
+        /// List of exception types that should trigger failover
+        /// </summary>
+        public HashSet<Type> FailoverExceptions { get; set; } = new()
+        {
+            typeof(AmazonS3Exception),
+            typeof(System.Net.Http.HttpRequestException),
+            typeof(System.Net.Sockets.SocketException),
+            typeof(System.Threading.Tasks.TaskCanceledException),
+            typeof(TimeoutException)
+        };
+
+        /// <summary>
+        /// List of S3 error codes that should trigger failover
+        /// </summary>
+        public HashSet<string> FailoverErrorCodes { get; set; } = new()
+        {
+            "ServiceUnavailable",
+            "SlowDown",
+            "InternalError",
+            "RequestTimeout"
+        };
+    }
+
+    /// <summary>
+    /// Result of a failover operation
+    /// </summary>
+    public class S3FailoverResult<T>
+    {
+        public T? Result { get; set; }
+        public bool UsedFailover { get; set; }
+        public int ClientIndex { get; set; } = 0;
+        public int AttemptsUsed { get; set; }
+        public Exception? LastException { get; set; }
+    }
+
+    /// <summary>
+    /// Helper class for S3 operations with automatic failover to multiple endpoints
+    /// </summary>
+    public class S3FailoverHelper
+    {
+        private readonly List<AmazonS3Client> clients;
+        private readonly S3FailoverConfig config;
+        private readonly ILogger? logger;
+
+        /// <summary>
+        /// Initialize with a list of S3 clients (first is primary, rest are failover endpoints)
+        /// </summary>
+        public S3FailoverHelper(
+            IEnumerable<AmazonS3Client> clients,
+            S3FailoverConfig? config = null,
+            ILogger? logger = null)
+        {
+            if (clients == null) throw new ArgumentNullException(nameof(clients));
+            this.clients = new List<AmazonS3Client>(clients);
+            if (this.clients.Count == 0) throw new ArgumentException("At least one S3 client is required", nameof(clients));
+            this.config = config ?? new S3FailoverConfig();
+            this.logger = logger;
+        }
+
+        /// <summary>
+        /// Initialize with params array of S3 clients
+        /// </summary>
+        public S3FailoverHelper(
+            S3FailoverConfig? config = null,
+            ILogger? logger = null,
+            params AmazonS3Client[] clients)
+        {
+            if (clients == null || clients.Length == 0)
+                throw new ArgumentException("At least one S3 client is required", nameof(clients));
+            this.clients = new List<AmazonS3Client>(clients);
+            this.config = config ?? new S3FailoverConfig();
+            this.logger = logger;
+        }
+
+        /// <summary>
+        /// Execute an S3 operation with automatic failover
+        /// </summary>
+        /// <param name="operation">The S3 operation to execute</param>
+        /// <param name="operationName">Name of the operation for logging</param>
+        /// <param name="isWriteOperation">Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default.</param>
+        public async Task<T> ExecuteWithFailoverAsync<T>(
+            Func<AmazonS3Client, Task<T>> operation,
+            string operationName = "S3Operation",
+            bool isWriteOperation = false)
+        {
+            var result = await ExecuteWithFailoverInternalAsync(operation, operationName, isWriteOperation);
+            if (result.Result == null)
+            {
+                throw result.LastException ?? new Exception($"Failed to execute {operationName} on all endpoints");
+            }
+            return result.Result;
+        }
+
+        /// <summary>
+        /// Execute an S3 operation with automatic failover and return detailed result
+        /// </summary>
+        /// <param name="operation">The S3 operation to execute</param>
+        /// <param name="operationName">Name of the operation for logging</param>
+        /// <param name="isWriteOperation">Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default.</param>
+        private async Task<S3FailoverResult<T>> ExecuteWithFailoverInternalAsync<T>(
+            Func<AmazonS3Client, Task<T>> operation,
+            string operationName = "S3Operation",
+            bool isWriteOperation = false)
+        {
+            var result = new S3FailoverResult<T>();
+            Exception? lastException = null;
+
+            // Determine max clients to try based on write operation flag
+            var maxClientsToTry = (isWriteOperation && !config.AllowWriteFailover) ? 1 : clients.Count;
+
+            if (isWriteOperation && !config.AllowWriteFailover && clients.Count > 1)
+            {
+                logger?.LogDebug(
+                    "Write operation {Operation} will only use primary endpoint. Failover is disabled for write operations.",
+                    operationName);
+            }
+
+            // Try each client in sequence (first is primary, rest are failovers)
+            for (int i = 0; i < maxClientsToTry; i++)
+            {
+                var client = clients[i];
+                var clientName = i == 0 ? "primary" : $"failover-{i}";
+                var isPrimary = i == 0;
+
+                if (!isPrimary && lastException != null)
+                {
+                    logger?.LogWarning(lastException,
+                        "Previous S3 endpoint failed for {Operation}. Attempting {ClientName} (endpoint {Index}/{Total}).",
+                        operationName, clientName, i + 1, maxClientsToTry);
+                }
+
+                var (success, value, exception, attempts) = await TryExecuteAsync(client, operation, operationName, clientName);
+                result.AttemptsUsed += attempts;
+
+                if (success && value != null)
+                {
+                    result.Result = value;
+                    result.UsedFailover = !isPrimary;
+                    result.ClientIndex = i;
+
+                    if (!isPrimary)
+                    {
+                        logger?.LogInformation(
+                            "Successfully failed over to {ClientName} S3 endpoint for {Operation}",
+                            clientName, operationName);
+                    }
+
+                    return result;
+                }
+
+                lastException = exception;
+
+                // If this is not the last client and should retry, log and continue
+                if (i < maxClientsToTry - 1 && ShouldFailover(exception))
+                {
+                    logger?.LogWarning(exception,
+                        "Endpoint {ClientName} failed for {Operation}. {Remaining} endpoint(s) remaining.",
+                        clientName, operationName, maxClientsToTry - i - 1);
+                }
+            }
+
+            // All clients failed
+            result.LastException = lastException;
+            logger?.LogError(lastException,
+                "All S3 endpoints failed for {Operation}. Total endpoints tried: {EndpointCount}, Total attempts: {Attempts}",
+                operationName, maxClientsToTry, result.AttemptsUsed);
+
+            return result;
+        }        /// <summary>
+                 /// Try to execute an operation with retries
+                 /// </summary>
+        private async Task<(bool success, T? value, Exception? exception, int attempts)> TryExecuteAsync<T>(
+            AmazonS3Client client,
+            Func<AmazonS3Client, Task<T>> operation,
+            string operationName,
+            string endpointName)
+        {
+            Exception? lastException = null;
+            int attempts = 0;
+
+            for (int retry = 0; retry <= config.MaxRetries; retry++)
+            {
+                attempts++;
+                try
+                {
+                    var result = await operation(client);
+                    return (true, result, null, attempts);
+                }
+                catch (Exception ex)
+                {
+                    lastException = ex;
+
+                    if (retry < config.MaxRetries && ShouldRetry(ex))
+                    {
+                        var delay = CalculateRetryDelay(retry);
+                        logger?.LogWarning(ex,
+                            "Attempt {Attempt}/{MaxAttempts} failed for {Operation} on {Endpoint}. Retrying in {Delay}ms",
+                            retry + 1, config.MaxRetries + 1, operationName, endpointName, delay);
+
+                        await Task.Delay(delay);
+                    }
+                    else
+                    {
+                        logger?.LogError(ex,
+                            "Operation {Operation} failed on {Endpoint} after {Attempts} attempts",
+                            operationName, endpointName, attempts);
+                        break;
+                    }
+                }
+            }
+
+            return (false, default, lastException, attempts);
+        }
+
+        /// <summary>
+        /// Determine if an exception should trigger a retry
+        /// </summary>
+        private bool ShouldRetry(Exception exception)
+        {
+            // Check if exception type is in the retry list
+            var exceptionType = exception.GetType();
+            if (config.FailoverExceptions.Contains(exceptionType))
+            {
+                // For S3 exceptions, check error codes
+                if (exception is AmazonS3Exception s3Exception)
+                {
+                    return config.FailoverErrorCodes.Contains(s3Exception.ErrorCode);
+                }
+                return true;
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Determine if an exception should trigger failover to secondary endpoint
+        /// </summary>
+        private bool ShouldFailover(Exception? exception)
+        {
+            if (exception == null) return false;
+            return ShouldRetry(exception);
+        }
+
+        /// <summary>
+        /// Calculate delay for retry based on retry attempt number
+        /// </summary>
+        private int CalculateRetryDelay(int retryAttempt)
+        {
+            if (!config.UseExponentialBackoff)
+            {
+                return config.RetryDelayMs;
+            }
+
+            // Exponential backoff: delay * 2^retryAttempt
+            return config.RetryDelayMs * (int)Math.Pow(2, retryAttempt);
+        }
+
+        /// <summary>
+        /// Execute a void operation with automatic failover
+        /// </summary>
+        /// <param name="operation">The S3 operation to execute</param>
+        /// <param name="operationName">Name of the operation for logging</param>
+        /// <param name="isWriteOperation">Whether this is a write operation (PUT/POST/DELETE). Write operations only use primary endpoint by default.</param>
+        public async Task ExecuteWithFailoverAsync(
+            Func<AmazonS3Client, Task> operation,
+            string operationName = "S3Operation",
+            bool isWriteOperation = false)
+        {
+            await ExecuteWithFailoverAsync<object?>(async (client) =>
+            {
+                await operation(client);
+                return null;
+            }, operationName, isWriteOperation);
+        }
+    }
+
+    public static class S3ClientFactory
+    {
+        public static List<AmazonS3Client> CreateS3Clients(
+            string serviceUrls,
+            string regions,
+            string accessKeyIds,
+            string secretKeys,
+            bool forcePathStyle = true)
+        {
+            if (string.IsNullOrWhiteSpace(serviceUrls))
+                return new List<AmazonS3Client>();
+
+            var urls = SplitAndTrim(serviceUrls);
+            var regionList = SplitAndTrim(regions);
+            var keyIds = SplitAndTrim(accessKeyIds);
+            var secrets = SplitAndTrim(secretKeys);
+
+            if (urls.Length != regionList.Length ||
+                urls.Length != keyIds.Length ||
+                urls.Length != secrets.Length)
+            {
+                throw new ArgumentException("All S3 configuration parameters must have the same number of values");
+            }
+
+            var clients = new List<AmazonS3Client>();
+
+            for (int i = 0; i < urls.Length; i++)
+            {
+                var url = urls[i];
+                if (string.IsNullOrWhiteSpace(url))
+                    continue;
+
+                // Get corresponding values from other arrays
+                var region = regionList[i];
+                var keyId = keyIds[i];
+                var secret = secrets[i];
+
+                // Validate that all required values are present
+                if (string.IsNullOrWhiteSpace(region) ||
+                    string.IsNullOrWhiteSpace(keyId) ||
+                    string.IsNullOrWhiteSpace(secret))
+                {
+                    System.Diagnostics.Debug.WriteLine(
+                        $"Skipping S3 client at index {i}: Missing required values (URL={url}, Region={region}, KeyId={keyId?.Length > 0}, Secret={secret?.Length > 0})");
+                    continue;
+                }
+
+                try
+                {
+                    var config = new AmazonS3Config
+                    {
+                        ServiceURL = url,
+                        AuthenticationRegion = region,
+                        ForcePathStyle = forcePathStyle,
+                        SignatureMethod = Amazon.Runtime.SigningAlgorithm.HmacSHA256,
+                        SignatureVersion = "4"
+                    };
+
+                    var client = new AmazonS3Client(keyId, secret, config);
+                    clients.Add(client);
+                }
+                catch (Exception ex)
+                {
+                    // Log configuration error but continue with other clients
+                    System.Diagnostics.Debug.WriteLine($"Failed to create S3 client for URL {url}: {ex.Message}");
+                }
+            }
+
+            return clients;
+        }
+
+        private static string[] SplitAndTrim(string? input)
+        {
+            if (string.IsNullOrWhiteSpace(input))
+                return Array.Empty<string>();
+
+            return input.Split(';', StringSplitOptions.None)
+                       .Select(s => s.Trim())
+                       .ToArray();
+        }
+    }
+}

Notesnook.API/Helpers/StorageHelper.cs

@@ -0,0 +1,79 @@
+using System;
+using System.Collections.Generic;
+using Notesnook.API.Models;
+using Streetwriters.Common.Enums;
+using Streetwriters.Common.Models;
+
+namespace Notesnook.API.Helpers
+{
+    class StorageHelper
+    {
+        const long MB = 1024 * 1024;
+        const long GB = 1024 * MB;
+        public readonly static Dictionary<SubscriptionPlan, long> MAX_STORAGE_PER_MONTH = new()
+            {
+                { SubscriptionPlan.FREE, 50L * MB },
+                { SubscriptionPlan.ESSENTIAL, GB },
+                { SubscriptionPlan.PRO, 10L * GB },
+                { SubscriptionPlan.EDUCATION, 10L * GB },
+                { SubscriptionPlan.BELIEVER, 25L * GB },
+                { SubscriptionPlan.LEGACY_PRO, -1 }
+            };
+        public readonly static Dictionary<SubscriptionPlan, long> MAX_FILE_SIZE = new()
+            {
+                { SubscriptionPlan.FREE, 10 * MB },
+                { SubscriptionPlan.ESSENTIAL, 100 * MB },
+                { SubscriptionPlan.PRO, 1L * GB },
+                { SubscriptionPlan.EDUCATION, 1L * GB },
+                { SubscriptionPlan.BELIEVER, 5L * GB },
+                { SubscriptionPlan.LEGACY_PRO, 512 * MB }
+            };
+
+        public static long GetStorageLimitForPlan(Subscription subscription)
+        {
+            return MAX_STORAGE_PER_MONTH[subscription.Plan];
+        }
+
+        public static long GetFileSizeLimitForPlan(Subscription subscription)
+        {
+            return MAX_FILE_SIZE[subscription.Plan];
+        }
+
+        public static bool IsStorageLimitReached(Subscription subscription, long limit)
+        {
+            var storageLimit = GetStorageLimitForPlan(subscription);
+            if (storageLimit == -1) return false;
+            return limit > storageLimit;
+        }
+
+        public static bool IsFileSizeExceeded(Subscription subscription, long fileSize)
+        {
+            var maxFileSize = MAX_FILE_SIZE[subscription.Plan];
+            return fileSize > maxFileSize;
+        }
+
+        public static Limit RolloverStorageLimit(Limit? limit)
+        {
+            var updatedAt = DateTimeOffset.FromUnixTimeMilliseconds(limit?.UpdatedAt ?? 0);
+            if (limit == null || DateTimeOffset.UtcNow.Year > updatedAt.Year || DateTimeOffset.UtcNow.Month > updatedAt.Month)
+            {
+                limit = new Limit { UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), Value = 0 };
+                return limit;
+            }
+            return limit;
+        }
+
+        private static readonly string[] sizes = ["B", "KB", "MB", "GB", "TB"];
+        public static string FormatBytes(long size)
+        {
+            int order = 0;
+            while (size >= 1024 && order < sizes.Length - 1)
+            {
+                order++;
+                size = size / 1024;
+            }
+
+            return String.Format("{0:0.##} {1}", size, sizes[order]);
+        }
+    }
+}

Notesnook.API/Helpers/UrlHelper.cs

@@ -0,0 +1,50 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using Notesnook.API.Models;
+using Streetwriters.Common;
+
+namespace Notesnook.API.Helpers
+{
+    public class UrlHelper
+    {
+        public static string ConstructPublishUrl(string slug)
+        {
+            var baseUrl = Constants.MONOGRAPH_PUBLIC_URL;
+            return $"{baseUrl}/{slug}";
+        }
+        public static string ConstructPublishUrl(Monograph monograph)
+        {
+            if (!string.IsNullOrEmpty(monograph.Slug))
+            {
+                return ConstructPublishUrl("s/" + monograph.Slug);
+            }
+            return ConstructPublishUrl(monograph.ItemId ?? monograph.Id);
+        }
+
+        public static string ConstructPublishUrl(MonographMetadata metadata)
+        {
+            if (!string.IsNullOrEmpty(metadata.PublishUrl))
+            {
+                return ConstructPublishUrl("s/" + metadata.PublishUrl);
+            }
+            return ConstructPublishUrl(metadata.PublishUrl ?? metadata.ItemId);
+        }
+    }
+}

Notesnook.API/Hubs/SyncHub.cs

@@ -1,473 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Security.Claims;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.SignalR;
-using MongoDB.Driver;
-using Notesnook.API.Authorization;
-using Notesnook.API.Interfaces;
-using Notesnook.API.Models;
-using Notesnook.API.Repositories;
-using Streetwriters.Common.Models;
-using Streetwriters.Data.Interfaces;
-
-namespace Notesnook.API.Hubs
-{
-    public struct RunningPush
-    {
-        public long Timestamp { get; set; }
-        public long Validity { get; set; }
-        public string ConnectionId { get; set; }
-    }
-    public interface ISyncHubClient
-    {
-        Task PushItems(SyncTransferItemV2 transferItem);
-        Task<bool> SendItems(SyncTransferItemV2 transferItem);
-        Task PushCompleted(long lastSynced);
-    }
-
-    public class GlobalSync
-    {
-        private const long PUSH_VALIDITY_EXTENSION_PERIOD = 16 * 1000; // 16 second
-        private const int PUSH_VALIDITY_PERIOD_PER_ITEM = 5 * 100; // 0.5 second
-        private const long BASE_PUSH_VALIDITY_PERIOD = 5 * 1000; // 5 seconds
-        private const long BASE_PUSH_VALIDITY_PERIOD_NEW = 16 * 1000; // 16 seconds
-        private readonly static Dictionary<string, List<RunningPush>> PushOperations = new();
-
-        public static void ClearPushOperations(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations.ToArray())
-                    if (push.ConnectionId == connectionId || !IsPushValid(push, now))
-                        operations.Remove(push);
-            }
-        }
-
-        public static bool IsPushing(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (push.ConnectionId == connectionId && IsPushValid(push, now)) return true;
-            }
-            return false;
-        }
-
-
-        public static bool IsUserPushing(string userId)
-        {
-            var count = 0;
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (IsPushValid(push, now)) ++count;
-            }
-            return count > 0;
-        }
-
-        public static void StartPush(string userId, string connectionId, long? totalItems = null)
-        {
-            if (IsPushing(userId, connectionId)) return;
-
-            if (!PushOperations.ContainsKey(userId))
-                PushOperations[userId] = new List<RunningPush>();
-
-            PushOperations[userId].Add(new RunningPush
-            {
-                ConnectionId = connectionId,
-                Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-                Validity = totalItems.HasValue ? BASE_PUSH_VALIDITY_PERIOD + (totalItems.Value * PUSH_VALIDITY_PERIOD_PER_ITEM) : BASE_PUSH_VALIDITY_PERIOD_NEW
-            });
-        }
-        public static void ExtendPush(string userId, string connectionId)
-        {
-            if (!IsPushing(userId, connectionId) || !PushOperations.ContainsKey(userId))
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var index = PushOperations[userId].FindIndex((push) => push.ConnectionId == connectionId);
-            if (index < 0)
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var pushOperation = PushOperations[userId][index];
-            pushOperation.Validity += PUSH_VALIDITY_EXTENSION_PERIOD;
-        }
-        private static bool IsPushValid(RunningPush push, long now)
-        {
-            return now < push.Timestamp + push.Validity;
-        }
-    }
-
-    [Authorize("Sync")]
-    public class SyncHub : Hub<ISyncHubClient>
-    {
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = new[] {
-            "settings",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "relation", // relations must sync at the end to prevent invalid state
-        };
-
-        public SyncHub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
-        {
-            Repositories = syncItemsRepositoryAccessor;
-            unit = unitOfWork;
-        }
-
-        public override async Task OnConnectedAsync()
-        {
-            var result = new SyncRequirement().IsAuthorized(Context.User, new PathString("/hubs/sync"));
-            if (!result.Succeeded)
-            {
-                var reason = result.AuthorizationFailure.FailureReasons.FirstOrDefault();
-                throw new HubException(reason?.Message ?? "Unauthorized");
-            }
-            var id = Context.User.FindFirstValue("sub");
-            await Groups.AddToGroupAsync(Context.ConnectionId, id);
-            await base.OnConnectedAsync();
-        }
-
-        public override async Task OnDisconnectedAsync(Exception exception)
-        {
-            try
-            {
-                await base.OnDisconnectedAsync(exception);
-            }
-            finally
-            {
-                var id = Context.User.FindFirstValue("sub");
-                GlobalSync.ClearPushOperations(id, Context.ConnectionId);
-            }
-        }
-
-        private Action<SyncItem, string, long> MapTypeToUpsertAction(string type)
-        {
-            return type switch
-            {
-                "attachment" => Repositories.Attachments.Upsert,
-                "note" => Repositories.Notes.Upsert,
-                "notebook" => Repositories.Notebooks.Upsert,
-                "content" => Repositories.Contents.Upsert,
-                "shortcut" => Repositories.Shortcuts.Upsert,
-                "reminder" => Repositories.Reminders.Upsert,
-                "relation" => Repositories.Relations.Upsert,
-                _ => null,
-            };
-        }
-
-        public async Task<long> InitializePush(SyncMetadata syncMetadata)
-        {
-            if (syncMetadata.LastSynced <= 0) throw new HubException("Last synced time cannot be zero or less than zero.");
-
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) return 0;
-
-            UserSettings userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            long dateSynced = Math.Max(syncMetadata.LastSynced, userSettings.LastSynced);
-
-            GlobalSync.StartPush(userId, Context.ConnectionId);
-
-
-            if (
-                (userSettings.VaultKey != null &&
-                syncMetadata.VaultKey != null &&
-                !userSettings.VaultKey.Equals(syncMetadata.VaultKey) &&
-                !syncMetadata.VaultKey.IsEmpty()) ||
-                (userSettings.VaultKey == null &&
-                syncMetadata.VaultKey != null &&
-                !syncMetadata.VaultKey.IsEmpty()))
-            {
-                userSettings.VaultKey = syncMetadata.VaultKey;
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-            }
-
-            return dateSynced;
-        }
-
-        public async Task<int> PushItems(SyncTransferItemV2 pushItem, long dateSynced)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) return 0;
-
-            SyncEventCounterSource.Log.Push();
-
-            try
-            {
-                var others = Clients.OthersInGroup(userId);
-                others.PushItems(pushItem);
-
-                GlobalSync.ExtendPush(userId, Context.ConnectionId);
-
-                if (pushItem.Type == "settings")
-                {
-                    var settings = pushItem.Items.First();
-                    if (settings == null) return 0;
-                    settings.Id = MongoDB.Bson.ObjectId.Parse(userId);
-                    settings.ItemId = userId;
-                    Repositories.LegacySettings.Upsert(settings, userId, dateSynced);
-                }
-                else
-                {
-                    var UpsertItem = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception("Invalid item type.");
-                    foreach (var item in pushItem.Items)
-                    {
-                        UpsertItem(item, userId, dateSynced);
-                    }
-                }
-
-                return await unit.Commit() ? 1 : 0;
-            }
-            catch (Exception ex)
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-                throw ex;
-            }
-        }
-
-        public async Task<bool> SyncCompleted(long dateSynced)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            try
-            {
-                UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-
-                long lastSynced = Math.Max(dateSynced, userSettings.LastSynced);
-
-                userSettings.LastSynced = lastSynced;
-
-                await this.Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-
-                await Clients.OthersInGroup(userId).PushCompleted(lastSynced);
-
-                return true;
-            }
-            finally
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-            }
-        }
-
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, long, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, long lastSyncedTimestamp, int size, long maxBytes, int skipChunks)
-        {
-            var chunksProcessed = 0;
-            for (int i = 0; i < collections.Length; i++)
-            {
-                var type = types[i];
-
-                using var cursor = await collections[i](userId, lastSyncedTimestamp, size);
-
-                var chunk = new List<SyncItem>();
-                long totalBytes = 0;
-                long METADATA_BYTES = 5 * 1024;
-
-                while (await cursor.MoveNextAsync())
-                {
-                    if (chunksProcessed++ < skipChunks) continue;
-                    foreach (var item in cursor.Current)
-                    {
-                        chunk.Add(item);
-                        totalBytes += item.Length + METADATA_BYTES;
-                        if (totalBytes >= maxBytes)
-                        {
-                            yield return new SyncTransferItemV2
-                            {
-                                Items = chunk,
-                                Type = type,
-                                Count = chunksProcessed
-                            };
-
-                            totalBytes = 0;
-                            chunk.Clear();
-                        }
-                    }
-                }
-                if (chunk.Count > 0)
-                {
-                    if (chunksProcessed++ < skipChunks) continue;
-                    yield return new SyncTransferItemV2
-                    {
-                        Items = chunk,
-                        Type = type,
-                        Count = chunksProcessed
-                    };
-                }
-            }
-        }
-
-        public Task<SyncMetadata> RequestFetch(long lastSyncedTimestamp)
-        {
-            return RequestResumableFetch(lastSyncedTimestamp);
-        }
-
-        public async Task<SyncMetadata> RequestResumableFetch(long lastSyncedTimestamp, int cursor = 0)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-
-            if (GlobalSync.IsUserPushing(userId))
-            {
-                throw new HubException("Cannot fetch data while another sync is in progress. Please try again later.");
-            }
-
-            SyncEventCounterSource.Log.Fetch();
-
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
-            {
-                throw new HubException($"Provided timestamp value is too large. Server timestamp: {userSettings.LastSynced} Sent timestamp: {lastSyncedTimestamp}. Please run a Force Sync to fix this issue.");
-            }
-            // var client = Clients.Caller;
-
-            if (lastSyncedTimestamp > 0 && userSettings.LastSynced == lastSyncedTimestamp)
-            {
-                return new SyncMetadata
-                {
-                    LastSynced = userSettings.LastSynced,
-                };
-            }
-
-            var isResumable = lastSyncedTimestamp == 0;
-            if (!isResumable) cursor = 0;
-
-            var chunks = PrepareChunks(
-                collections: new[] {
-                    Repositories.LegacySettings.FindItemsSyncedAfter,
-                    Repositories.Attachments.FindItemsSyncedAfter,
-                    Repositories.Notes.FindItemsSyncedAfter,
-                    Repositories.Notebooks.FindItemsSyncedAfter,
-                    Repositories.Contents.FindItemsSyncedAfter,
-                    Repositories.Shortcuts.FindItemsSyncedAfter,
-                    Repositories.Reminders.FindItemsSyncedAfter,
-                    Repositories.Relations.FindItemsSyncedAfter,
-                },
-                types: CollectionKeys,
-                userId,
-                lastSyncedTimestamp,
-                size: 1000,
-                maxBytes: 7 * 1024 * 1024,
-                skipChunks: cursor
-            );
-
-            await foreach (var chunk in chunks)
-            {
-                _ = await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10));
-            }
-
-            return new SyncMetadata
-            {
-                VaultKey = userSettings.VaultKey,
-                LastSynced = userSettings.LastSynced,
-            };
-        }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct BatchedSyncTransferItem
-    {
-        [MessagePack.Key("lastSynced")]
-        public long LastSynced { get; set; }
-
-        [MessagePack.Key("items")]
-        public string[] Items { get; set; }
-
-        [MessagePack.Key("types")]
-        public string[] Types { get; set; }
-
-        [MessagePack.Key("total")]
-        public int Total { get; set; }
-
-        [MessagePack.Key("current")]
-        public int Current { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncTransferItem
-    {
-        [MessagePack.Key("synced")]
-        public bool Synced { get; set; }
-
-        [MessagePack.Key("lastSynced")]
-        public long LastSynced { get; set; }
-
-        [MessagePack.Key("item")]
-        public string Item { get; set; }
-
-        [MessagePack.Key("itemType")]
-        public string ItemType { get; set; }
-
-        [MessagePack.Key("total")]
-        public int Total { get; set; }
-
-        [MessagePack.Key("current")]
-        public int Current { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncTransferItemV2
-    {
-        [MessagePack.Key("items")]
-        [JsonPropertyName("items")]
-        public IEnumerable<SyncItem> Items { get; set; }
-
-        [MessagePack.Key("type")]
-        [JsonPropertyName("type")]
-        public string Type { get; set; }
-        [MessagePack.Key("count")]
-        [JsonPropertyName("count")]
-        public int Count { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncMetadata
-    {
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
-
-        [MessagePack.Key("lastSynced")]
-        [JsonPropertyName("lastSynced")]
-        public long LastSynced { get; set; }
-        // [MessagePack.Key("total")]
-        // public long TotalItems { get; set; }
-    }
-}

Notesnook.API/Hubs/SyncV2Hub.cs

@@ -18,17 +18,22 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
+using System.Collections.Frozen;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Linq.Expressions;
 using System.Security.Claims;
 using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using Notesnook.API.Authorization;
+using Notesnook.API.Extensions;
+using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Services;
@@ -40,33 +45,71 @@ namespace Notesnook.API.Hubs
     {
         Task<bool> SendItems(SyncTransferItemV2 transferItem);
         Task<bool> SendVaultKey(EncryptedData vaultKey);
-        Task<bool> SendMonographs(IEnumerable<Monograph> monographs);
+        Task<bool> SendMonographs(IEnumerable<MonographMetadata> monographs);
+        Task<bool> SendInboxItems(IEnumerable<InboxSyncItem> inboxItems);
         Task PushCompleted();
+        Task PushCompletedV2(string deviceId);
     }
 
-    [Authorize("Sync")]
+    [Authorize]
     public class SyncV2Hub : Hub<ISyncV2HubClient>
     {
         private ISyncItemsRepositoryAccessor Repositories { get; }
+        private SyncDeviceService SyncDeviceService { get; }
         private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = [
-            "settingitem",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "color",
-            "tag",
-            "vault",
-            "relation", // relations must sync at the end to prevent invalid state
-        ];
+        private readonly FrozenDictionary<string, Action<IEnumerable<SyncItem>, string>> UpsertActionsMap;
+        private readonly CollectionDef[] BaseCollectionDefs;
+        private readonly CollectionDef[] V4CollectionDefs;
+        ILogger<SyncV2Hub> Logger { get; }
 
-        public SyncV2Hub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
+        public SyncV2Hub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork, SyncDeviceService syncDeviceService, ILogger<SyncV2Hub> logger)
         {
+            Logger = logger;
             Repositories = syncItemsRepositoryAccessor;
             unit = unitOfWork;
+            SyncDeviceService = syncDeviceService;
+
+            BaseCollectionDefs = [
+                new("settingitem", Repositories.Settings.FindItemsById),
+                new("attachment", Repositories.Attachments.FindItemsById),
+                new("note", Repositories.Notes.FindItemsById),
+                new("notebook", Repositories.Notebooks.FindItemsById),
+                new("content", Repositories.Contents.FindItemsById),
+                new("shortcut", Repositories.Shortcuts.FindItemsById),
+                new("reminder", Repositories.Reminders.FindItemsById),
+                new("color", Repositories.Colors.FindItemsById),
+                new("tag", Repositories.Tags.FindItemsById),
+                new("vault", Repositories.Vaults.FindItemsById),
+                new("relation", Repositories.Relations.FindItemsById), // relations must sync at the end to prevent invalid state
+            ];
+            V4CollectionDefs = [
+                new("settingitem", Repositories.Settings.FindItemsById),
+                new("attachment", Repositories.Attachments.FindItemsById),
+                new("note", Repositories.Notes.FindItemsById),
+                new("notebook", Repositories.Notebooks.FindItemsById),
+                new("content", Repositories.Contents.FindItemsById),
+                new("shortcut", Repositories.Shortcuts.FindItemsById),
+                new("reminder", Repositories.Reminders.FindItemsById),
+                new("color", Repositories.Colors.FindItemsById),
+                new("tag", Repositories.Tags.FindItemsById),
+                new("vault", Repositories.Vaults.FindItemsById),
+                new("inboxitemhistory", Repositories.InboxItemsHistory.FindItemsById),
+                new("relation", Repositories.Relations.FindItemsById), // relations must sync at the end to prevent invalid state
+            ];
+            UpsertActionsMap = new Dictionary<string, Action<IEnumerable<SyncItem>, string>> {
+                { "settingitem", Repositories.Settings.UpsertMany },
+                { "attachment", Repositories.Attachments.UpsertMany },
+                { "note", Repositories.Notes.UpsertMany },
+                { "notebook", Repositories.Notebooks.UpsertMany },
+                { "content", Repositories.Contents.UpsertMany },
+                { "shortcut", Repositories.Shortcuts.UpsertMany },
+                { "reminder", Repositories.Reminders.UpsertMany },
+                { "relation", Repositories.Relations.UpsertMany },
+                { "color", Repositories.Colors.UpsertMany },
+                { "vault", Repositories.Vaults.UpsertMany },
+                { "tag", Repositories.Tags.UpsertMany },
+                { "inboxitemhistory", Repositories.InboxItemsHistory.UpsertMany },
+            }.ToFrozenDictionary();
         }
 
         public override async Task OnConnectedAsync()
@@ -74,97 +117,84 @@ namespace Notesnook.API.Hubs
             var result = new SyncRequirement().IsAuthorized(Context.User, new PathString("/hubs/sync/v2"));
             if (!result.Succeeded)
             {
-                var reason = result.AuthorizationFailure.FailureReasons.FirstOrDefault();
+                var reason = result.AuthorizationFailure?.FailureReasons.FirstOrDefault();
                 throw new HubException(reason?.Message ?? "Unauthorized");
             }
-            var id = Context.User.FindFirstValue("sub");
+            var id = Context.User?.FindFirstValue("sub") ?? throw new HubException("User not found.");
             await Groups.AddToGroupAsync(Context.ConnectionId, id);
             await base.OnConnectedAsync();
         }
 
-        private Action<IEnumerable<SyncItem>, string, long> MapTypeToUpsertAction(string type)
+        public override async Task OnDisconnectedAsync(Exception? exception)
         {
-            return type switch
+            if (exception != null)
             {
-                "settingitem" => Repositories.Settings.UpsertMany,
-                "attachment" => Repositories.Attachments.UpsertMany,
-                "note" => Repositories.Notes.UpsertMany,
-                "notebook" => Repositories.Notebooks.UpsertMany,
-                "content" => Repositories.Contents.UpsertMany,
-                "shortcut" => Repositories.Shortcuts.UpsertMany,
-                "reminder" => Repositories.Reminders.UpsertMany,
-                "relation" => Repositories.Relations.UpsertMany,
-                "color" => Repositories.Colors.UpsertMany,
-                "vault" => Repositories.Vaults.UpsertMany,
-                "tag" => Repositories.Tags.UpsertMany,
-                _ => null,
-            };
+                Logger.LogWarning(exception, "Connection {ConnectionId} disconnected with error (server-side drop)", Context.ConnectionId);
+            }
+            else
+            {
+                Logger.LogInformation("Connection {ConnectionId} disconnected cleanly (client-initiated)", Context.ConnectionId);
+            }
+            await base.OnDisconnectedAsync(exception);
         }
 
-        private Func<string, IEnumerable<string>, bool, int, Task<IAsyncCursor<SyncItem>>> MapTypeToFindItemsAction(string type)
-        {
-            return type switch
-            {
-                "settingitem" => Repositories.Settings.FindItemsById,
-                "attachment" => Repositories.Attachments.FindItemsById,
-                "note" => Repositories.Notes.FindItemsById,
-                "notebook" => Repositories.Notebooks.FindItemsById,
-                "content" => Repositories.Contents.FindItemsById,
-                "shortcut" => Repositories.Shortcuts.FindItemsById,
-                "reminder" => Repositories.Reminders.FindItemsById,
-                "relation" => Repositories.Relations.FindItemsById,
-                "color" => Repositories.Colors.FindItemsById,
-                "vault" => Repositories.Vaults.FindItemsById,
-                "tag" => Repositories.Tags.FindItemsById,
-                _ => null,
-            };
-        }
 
         public async Task<int> PushItems(string deviceId, SyncTransferItemV2 pushItem)
         {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
+            var userId = Context.User?.FindFirstValue("sub") ?? throw new HubException("Please login to sync.");
 
             SyncEventCounterSource.Log.PushV2();
 
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
+            var stopwatch = Stopwatch.StartNew();
             try
             {
-
-                var UpsertItems = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception($"Invalid item type: {pushItem.Type}.");
-                UpsertItems(pushItem.Items, userId, 1);
+                var UpsertItems = UpsertActionsMap[pushItem.Type] ?? throw new Exception($"Invalid item type: {pushItem.Type}.");
+                UpsertItems(pushItem.Items, userId);
 
                 if (!await unit.Commit()) return 0;
 
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).AddIdsToOtherDevices(pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
+                await SyncDeviceService.AddIdsToOtherDevicesAsync(userId, deviceId, pushItem.Items.Select((i) => new ItemKey(i.ItemId, pushItem.Type)));
+
+                // we need to delete the inbox items from the inbox collection
+                // after syncing to prevent them from being sent again in the 
+                // next fetch.
+                var itemIds = pushItem.Items.Select(i => i.ItemId).ToList();
+                await Repositories.InboxItems.DeleteManyAsync(i => i.UserId == userId && itemIds.Contains(i.ItemId));
+
                 return 1;
             }
             finally
             {
-                stopwatch.Stop();
                 SyncEventCounterSource.Log.RecordPushDuration(stopwatch.ElapsedMilliseconds);
             }
         }
 
         public async Task<bool> PushCompleted()
         {
-            var userId = Context.User.FindFirstValue("sub");
+            var userId = Context.User?.FindFirstValue("sub") ?? throw new HubException("User not found.");
             await Clients.OthersInGroup(userId).PushCompleted();
             return true;
         }
 
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, string[], bool, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, string[] ids, int size, bool resetSync, long maxBytes)
+        public async Task<bool> PushCompletedV2(string deviceId)
+        {
+            var userId = Context.User?.FindFirstValue("sub") ?? throw new HubException("User not found.");
+            await Clients.OthersInGroup(userId).PushCompleted();
+            await Clients.OthersInGroup(userId).PushCompletedV2(deviceId);
+            return true;
+        }
+
+        private async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(string userId, HashSet<ItemKey> ids, int size, bool resetSync, long maxBytes, CollectionDef[] collectionDefs)
         {
             var itemsProcessed = 0;
-            for (int i = 0; i < collections.Length; i++)
+            foreach (var def in collectionDefs)
             {
-                var type = types[i];
+                var type = def.Key;
 
-                var filteredIds = ids.Where((id) => id.EndsWith($":{type}")).Select((id) => id.Split(":")[0]).ToArray();
+                var filteredIds = ids.Where((id) => id.Type == type).Select((id) => id.ItemId).ToArray();
                 if (!resetSync && filteredIds.Length == 0) continue;
 
-                using var cursor = await collections[i](userId, filteredIds, resetSync, size);
+                using var cursor = await def.FindItems(userId, filteredIds, resetSync, size);
 
                 var chunk = new List<SyncItem>();
                 long totalBytes = 0;
@@ -206,80 +236,112 @@ namespace Notesnook.API.Hubs
 
         public async Task<SyncV2Metadata> RequestFetch(string deviceId)
         {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
+            return await HandleRequestFetch(deviceId, false, false, BaseCollectionDefs);
+        }
+
+        public async Task<SyncV2Metadata> RequestFetchV2(string deviceId)
+        {
+            return await HandleRequestFetch(deviceId, true, false, BaseCollectionDefs);
+        }
+
+        public async Task<SyncV2Metadata> RequestFetchV3(string deviceId)
+        {
+            return await HandleRequestFetch(deviceId, true, true, BaseCollectionDefs);
+        }
+
+        public async Task<SyncV2Metadata> RequestFetchV4(string deviceId)
+        {
+            return await HandleRequestFetch(deviceId, true, true, V4CollectionDefs);
+        }
+
+        private async Task<SyncV2Metadata> HandleRequestFetch(string deviceId, bool includeMonographs, bool includeInboxItems, CollectionDef[] collectionDefs)
+        {
+            var userId = Context.User?.FindFirstValue("sub") ?? throw new HubException("Please login to sync.");
 
             SyncEventCounterSource.Log.FetchV2();
 
-            var device = new SyncDevice(userId, deviceId);
-            var deviceService = new SyncDeviceService(device);
-            if (!deviceService.IsDeviceRegistered()) deviceService.RegisterDevice();
+            var device = await SyncDeviceService.GetDeviceAsync(userId, deviceId);
+            if (device == null)
+                device = await SyncDeviceService.RegisterDeviceAsync(userId, deviceId);
+            else
+                await SyncDeviceService.UpdateLastAccessTimeAsync(userId, deviceId);
 
-            device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-
-            var isResetSync = deviceService.IsSyncReset();
-            if (!deviceService.IsUnsynced() &&
-                !deviceService.IsSyncPending() &&
-                !isResetSync)
-                return new SyncV2Metadata { Synced = true };
-
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
+            var stopwatch = Stopwatch.StartNew();
             try
             {
-                string[] ids = deviceService.FetchUnsyncedIds();
+                var ids = await SyncDeviceService.FetchUnsyncedIdsAsync(userId, deviceId);
+                if (!device.IsSyncReset && ids.Count == 0)
+                    return new SyncV2Metadata { Synced = true };
 
                 var chunks = PrepareChunks(
-                    collections: [
-                        Repositories.Settings.FindItemsById,
-                    Repositories.Attachments.FindItemsById,
-                    Repositories.Notes.FindItemsById,
-                    Repositories.Notebooks.FindItemsById,
-                    Repositories.Contents.FindItemsById,
-                    Repositories.Shortcuts.FindItemsById,
-                    Repositories.Reminders.FindItemsById,
-                    Repositories.Colors.FindItemsById,
-                    Repositories.Tags.FindItemsById,
-                    Repositories.Vaults.FindItemsById,
-                    Repositories.Relations.FindItemsById,
-                    ],
-                    types: CollectionKeys,
                     userId,
                     ids,
-                    size: 1000,
-                    resetSync: isResetSync,
-                    maxBytes: 7 * 1024 * 1024
+                    size: 100,
+                    resetSync: device.IsSyncReset,
+                    maxBytes: 3 * 1024 * 1024,
+                    collectionDefs
                 );
 
-                var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId.Equals(userId));
-                if (userSettings.VaultKey != null)
-                {
-                    if (!await Clients.Caller.SendVaultKey(userSettings.VaultKey).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected vault key.");
-                }
-
-
                 await foreach (var chunk in chunks)
                 {
                     if (!await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected sent items.");
 
-                    if (!isResetSync)
+                    if (!device.IsSyncReset)
                     {
-                        var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
-                        ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
-                        deviceService.WritePendingIds(ids);
+                        ids.ExceptWith(chunk.Items.Select(i => new ItemKey(i.ItemId, chunk.Type)));
+                        await SyncDeviceService.WritePendingIdsAsync(userId, deviceId, ids);
                     }
                 }
 
-                var unsyncedMonographs = ids.Where((id) => id.EndsWith(":monograph")).ToHashSet();
-                var unsyncedMonographIds = unsyncedMonographs.Select((id) => id.Split(":")[0]).ToArray();
-                var userMonographs = isResetSync
-                    ? await Repositories.Monographs.FindAsync(m => m.UserId == userId)
-                    : await Repositories.Monographs.FindAsync(m => m.UserId == userId && unsyncedMonographIds.Contains(m.ItemId));
+                if (includeMonographs)
+                {
+                    var unsyncedMonographIds = ids.Where(k => k.Type == "monograph").Select(k => k.ItemId);
+                    FilterDefinition<Monograph> filter = device.IsSyncReset
+                        ? Builders<Monograph>.Filter.Eq(m => m.UserId, userId)
+                        : Builders<Monograph>.Filter.And(
+                            Builders<Monograph>.Filter.Eq(m => m.UserId, userId),
+                            Builders<Monograph>.Filter.Or(
+                                Builders<Monograph>.Filter.In(m => m.ItemId, unsyncedMonographIds),
+                                Builders<Monograph>.Filter.In("_id", unsyncedMonographIds)
+                            )
+                        );
+                    var userMonographs = await Repositories.Monographs.Collection
+                        .Find(filter)
+                        .Project((m) => new MonographMetadata
+                        {
+                            DatePublished = m.DatePublished,
+                            Deleted = m.Deleted,
+                            Password = m.Password,
+                            SelfDestruct = m.SelfDestruct,
+                            Title = m.Title,
+                            ItemId = m.ItemId ?? m.Id.ToString(),
+                            PublishUrl = m.Slug // this will be converted to full url in the end, but we only need slug for now
+                        })
+                        .ToListAsync();
 
-                if (userMonographs.Any() && !await Clients.Caller.SendMonographs(userMonographs).WaitAsync(TimeSpan.FromMinutes(10)))
-                    throw new HubException("Client rejected monographs.");
+                    userMonographs = userMonographs.Select((p) =>
+                    {
+                        p.PublishUrl = UrlHelper.ConstructPublishUrl(p);
+                        return p;
+                    }).ToList();
 
-                deviceService.Reset();
+                    if (userMonographs.Count > 0 && !await Clients.Caller.SendMonographs(userMonographs).WaitAsync(TimeSpan.FromMinutes(10)))
+                        throw new HubException("Client rejected monographs.");
+                }
+
+                if (includeInboxItems)
+                {
+                    var unsyncedInboxItemIds = ids.Where(k => k.Type == "inbox_item").Select(k => k.ItemId);
+                    var userInboxItems = device.IsSyncReset
+                        ? await Repositories.InboxItems.FindAsync(m => m.UserId == userId)
+                        : await Repositories.InboxItems.FindAsync(m => m.UserId == userId && unsyncedInboxItemIds.Contains(m.ItemId));
+                    if (userInboxItems.Any() && !await Clients.Caller.SendInboxItems(userInboxItems).WaitAsync(TimeSpan.FromMinutes(10)))
+                    {
+                        throw new HubException("Client rejected inbox items.");
+                    }
+                }
+
+                await SyncDeviceService.ResetAsync(userId, deviceId);
 
                 return new SyncV2Metadata
                 {
@@ -288,10 +350,14 @@ namespace Notesnook.API.Hubs
             }
             finally
             {
-                stopwatch.Stop();
                 SyncEventCounterSource.Log.RecordFetchDuration(stopwatch.ElapsedMilliseconds);
             }
         }
+
+        private record CollectionDef(
+            string Key,
+            Func<string, IEnumerable<string>, bool, int, Task<IAsyncCursor<SyncItem>>> FindItems
+        );
     }
 
     [MessagePack.MessagePackObject]
@@ -303,7 +369,7 @@ namespace Notesnook.API.Hubs
     }
 
     [MessagePack.MessagePackObject]
-    public struct SyncV2TransferItem
+    public struct SyncTransferItemV2
     {
         [MessagePack.Key("items")]
         [JsonPropertyName("items")]
@@ -312,13 +378,8 @@ namespace Notesnook.API.Hubs
         [MessagePack.Key("type")]
         [JsonPropertyName("type")]
         public string Type { get; set; }
-
-        [MessagePack.Key("final")]
-        [JsonPropertyName("final")]
-        public bool Final { get; set; }
-
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
+        [MessagePack.Key("count")]
+        [JsonPropertyName("count")]
+        public int Count { get; set; }
     }
 }

Notesnook.API/Interfaces/IS3Service.cs

@@ -1,40 +1,39 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Threading;
-using System.Threading.Tasks;
-using Amazon.S3.Model;
-using Notesnook.API.Models;
-using Notesnook.API.Models.Responses;
-using Streetwriters.Common.Interfaces;
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IS3Service
-    {
-        Task DeleteObjectAsync(string userId, string name);
-        Task DeleteDirectoryAsync(string userId);
-        Task<long> GetObjectSizeAsync(string userId, string name);
-        string GetUploadObjectUrl(string userId, string name);
-        string GetDownloadObjectUrl(string userId, string name);
-        Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string uploadId = null);
-        Task AbortMultipartUploadAsync(string userId, string name, string uploadId);
-        Task CompleteMultipartUploadAsync(string userId, CompleteMultipartUploadRequest uploadRequest);
-    }
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Threading.Tasks;
+using Amazon.S3.Model;
+using Notesnook.API.Models;
+
+namespace Notesnook.API.Interfaces
+{
+    public interface IS3Service
+    {
+        Task DeleteObjectAsync(string userId, string name);
+        Task DeleteObjectsAsync(string userId, string[] names);
+        Task DeleteDirectoryAsync(string userId);
+        Task<long> GetObjectSizeAsync(string userId, string name);
+        Task<string?> GetUploadObjectUrlAsync(string userId, string name);
+        Task<string?> GetInternalUploadObjectUrlAsync(string userId, string name);
+        Task<string?> GetDownloadObjectUrlAsync(string userId, string name);
+        Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string? uploadId = null);
+        Task AbortMultipartUploadAsync(string userId, string name, string uploadId);
+        Task CompleteMultipartUploadAsync(string userId, CompleteMultipartUploadRequest uploadRequest);
+    }
 }

Notesnook.API/Interfaces/ISyncItemsRepositoryAccessor.cs

@@ -38,7 +38,12 @@ namespace Notesnook.API.Interfaces
         SyncItemsRepository Colors { get; }
         SyncItemsRepository Vaults { get; }
         SyncItemsRepository Tags { get; }
+        SyncItemsRepository InboxItemsHistory { get; }
         Repository<UserSettings> UsersSettings { get; }
         Repository<Monograph> Monographs { get; }
+        Repository<InboxApiKey> InboxApiKey { get; }
+        Repository<InboxSyncItem> InboxItems { get; }
+        Repository<SyncDevice> SyncDevices { get; }
+        Repository<DeviceIdsChunk> DeviceIdsChunks { get; }
     }
 }

Notesnook.API/Interfaces/IUserService.cs

@@ -20,14 +20,15 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using System.Threading.Tasks;
 using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
+using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Interfaces
 {
     public interface IUserService
     {
-        Task CreateUserAsync();
+        Task<SignupResponse> CreateUserAsync(SignupForm form);
         Task DeleteUserAsync(string userId);
-        Task DeleteUserAsync(string userId, string jti, string password);
+        Task DeleteUserAsync(string userId, string? jti, string password);
         Task<bool> ResetUserAsync(string userId, bool removeAttachments);
         Task<UserResponse> GetUserAsync(string userId);
         Task SetUserKeysAsync(string userId, UserKeys keys);

Notesnook.API/Jobs/DeviceCleanupJob.cs

@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using Notesnook.API.Interfaces;
+using Notesnook.API.Models;
+using Notesnook.API.Services;
+using Quartz;
+
+namespace Notesnook.API.Jobs
+{
+    public class DeviceCleanupJob(ISyncItemsRepositoryAccessor repositories) : IJob
+    {
+        public async Task Execute(IJobExecutionContext context)
+        {
+            var cutoffDate = DateTimeOffset.UtcNow.AddMonths(-1).ToUnixTimeMilliseconds();
+            var deviceFilter = Builders<SyncDevice>.Filter.Lt(x => x.LastAccessTime, cutoffDate);
+
+            using var cursor = await repositories.SyncDevices.Collection.Find(deviceFilter, new FindOptions { BatchSize = 1000 })
+                .Project(x => x.DeviceId)
+                .ToCursorAsync();
+
+            var deleteModels = new List<WriteModel<DeviceIdsChunk>>();
+            while (await cursor.MoveNextAsync())
+            {
+                if (!cursor.Current.Any()) continue;
+                deleteModels.Add(new DeleteManyModel<DeviceIdsChunk>(Builders<DeviceIdsChunk>.Filter.In(x => x.DeviceId, cursor.Current)));
+            }
+
+            if (deleteModels.Count > 0)
+            {
+                var bulkOptions = new BulkWriteOptions { IsOrdered = false };
+                await repositories.DeviceIdsChunks.Collection.BulkWriteAsync(deleteModels, bulkOptions);
+            }
+
+            await repositories.SyncDevices.Collection.DeleteManyAsync(deviceFilter);
+        }
+    }
+}

Notesnook.API/Models/Announcement.cs

@@ -40,7 +40,7 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("type")]
         [BsonElement("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("timestamp")]
         [BsonElement("timestamp")]
@@ -48,7 +48,7 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("platforms")]
         [BsonElement("platforms")]
-        public string[] Platforms { get; set; }
+        public required string[] Platforms { get; set; }
 
         [JsonPropertyName("isActive")]
         [BsonElement("isActive")]
@@ -56,7 +56,7 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("userTypes")]
         [BsonElement("userTypes")]
-        public string[] UserTypes { get; set; }
+        public required string[] UserTypes { get; set; }
 
         [JsonPropertyName("appVersion")]
         [BsonElement("appVersion")]
@@ -64,63 +64,63 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("body")]
         [BsonElement("body")]
-        public BodyComponent[] Body { get; set; }
+        public required BodyComponent[] Body { get; set; }
 
         [JsonIgnore]
         [BsonElement("userIds")]
-        public string[] UserIds { get; set; }
+        public string[]? UserIds { get; set; }
 
 
         [Obsolete]
         [JsonPropertyName("title")]
         [DataMember(Name = "title")]
         [BsonElement("title")]
-        public string Title { get; set; }
+        public string? Title { get; set; }
 
         [Obsolete]
         [JsonPropertyName("description")]
         [BsonElement("description")]
-        public string Description { get; set; }
+        public string? Description { get; set; }
 
         [Obsolete]
         [JsonPropertyName("callToActions")]
         [BsonElement("callToActions")]
-        public CallToAction[] CallToActions { get; set; }
+        public CallToAction[]? CallToActions { get; set; }
     }
 
     public class BodyComponent
     {
         [JsonPropertyName("type")]
         [BsonElement("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("platforms")]
         [BsonElement("platforms")]
-        public string[] Platforms { get; set; }
+        public string[]? Platforms { get; set; }
 
         [JsonPropertyName("style")]
         [BsonElement("style")]
-        public Style Style { get; set; }
+        public Style? Style { get; set; }
 
         [JsonPropertyName("src")]
         [BsonElement("src")]
-        public string Src { get; set; }
+        public string? Src { get; set; }
 
         [JsonPropertyName("text")]
         [BsonElement("text")]
-        public string Text { get; set; }
+        public string? Text { get; set; }
 
         [JsonPropertyName("value")]
         [BsonElement("value")]
-        public string Value { get; set; }
+        public string? Value { get; set; }
 
         [JsonPropertyName("items")]
         [BsonElement("items")]
-        public BodyComponent[] Items { get; set; }
+        public BodyComponent[]? Items { get; set; }
 
         [JsonPropertyName("actions")]
         [BsonElement("actions")]
-        public CallToAction[] Actions { get; set; }
+        public required CallToAction[] Actions { get; set; }
     }
 
     public class Style
@@ -135,25 +135,25 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("textAlign")]
         [BsonElement("textAlign")]
-        public string TextAlign { get; set; }
+        public string? TextAlign { get; set; }
     }
 
     public class CallToAction
     {
         [JsonPropertyName("type")]
         [BsonElement("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("platforms")]
         [BsonElement("platforms")]
-        public string[] Platforms { get; set; }
+        public string[]? Platforms { get; set; }
 
         [JsonPropertyName("data")]
         [BsonElement("data")]
-        public string Data { get; set; }
+        public string? Data { get; set; }
 
         [JsonPropertyName("title")]
         [BsonElement("title")]
-        public string Title { get; set; }
+        public string? Title { get; set; }
     }
 }

Notesnook.API/Models/ChangePasswordForm.cs

@@ -0,0 +1,24 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Notesnook.API.Models
+{
+    public class ChangePasswordForm
+    {
+        public string? OldPassword
+        {
+            get; set;
+        }
+
+        [Required]
+        public required string NewPassword
+        {
+            get; set;
+        }
+
+        [Required]
+        public required UserKeys UserKeys
+        {
+            get; set;
+        }
+    }
+}

Notesnook.API/Models/CompleteMultipartUploadRequestWrapper.cs

@@ -5,9 +5,9 @@ namespace Notesnook.API.Models;
 
 public class CompleteMultipartUploadRequestWrapper
 {
-    public string Key { get; set; }
-    public List<PartETagWrapper> PartETags { get; set; }
-    public string UploadId { get; set; }
+    public required string Key { get; set; }
+    public required List<PartETagWrapper> PartETags { get; set; }
+    public required string UploadId { get; set; }
 
     public CompleteMultipartUploadRequest ToRequest()
     {

Notesnook.API/Models/ContentSanitizationLevel.cs

@@ -1,42 +1,38 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using Notesnook.API.Models;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IUserSettings : IDocument
-    {
-        string UserId { get; set; }
-
-        long LastSynced
-        {
-            get; set;
-        }
-
-        EncryptedData VaultKey
-        {
-            get; set;
-        }
-
-        string Salt { get; set; }
-    }
-}
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Notesnook.API.Models
+{
+    public enum ContentSanitizationLevel
+    {
+        Unknown = 0,
+        /// <summary>
+        /// Full sanitization applied: links, iframes, images, and other embeds are stripped.
+        /// Applied to monographs published by free-tier users.
+        /// </summary>
+        Full = 1,
+
+        /// <summary>
+        /// Partial sanitization: only unsafe/malicious URLs are removed; rich content is preserved.
+        /// Applied to monographs published by subscribed users. Requires re-sanitization if the
+        /// publisher's subscription lapses.
+        /// </summary>
+        Partial = 2
+    }
+}

Notesnook.API/Models/DeleteAccountForm.cs

@@ -5,7 +5,7 @@ namespace Notesnook.API.Models
     public class DeleteAccountForm
     {
         [Required]
-        public string Password
+        public required string Password
         {
             get; set;
         }

Notesnook.API/Models/DeleteBulkRequest.cs

@@ -1,33 +1,25 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using Notesnook.API.Models;
-using Streetwriters.Common.Interfaces;
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IMonograph : IDocument
-    {
-        string Title { get; set; }
-        string UserId { get; set; }
-        byte[] CompressedContent { get; set; }
-        EncryptedData EncryptedContent { get; set; }
-        long DatePublished { get; set; }
-    }
-}
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Notesnook.API.Models;
+
+public class DeleteBulkObjectsRequest
+{
+    public required string[] Names { get; set; }
+}

Notesnook.API/Models/DeviceIdsChunk.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Notesnook.API.Models
+{
+    public class DeviceIdsChunk
+    {
+        [BsonId]
+        public ObjectId Id { get; set; }
+        public required string UserId { get; set; }
+        public required string DeviceId { get; set; }
+        public required string Key { get; set; }
+        public required string[] Ids { get; set; } = [];
+    }
+}

Notesnook.API/Models/EncryptedData.cs

@@ -26,25 +26,19 @@ using System.Text.Json.Serialization;
 namespace Notesnook.API.Models
 {
     [MessagePack.MessagePackObject]
-    public class EncryptedData : IEncrypted
+    public class EncryptedData
     {
         [MessagePack.Key("iv")]
         [JsonPropertyName("iv")]
         [BsonElement("iv")]
         [DataMember(Name = "iv")]
-        public string IV
-        {
-            get; set;
-        }
+        public required string IV { get; set; }
 
         [MessagePack.Key("cipher")]
         [JsonPropertyName("cipher")]
         [BsonElement("cipher")]
         [DataMember(Name = "cipher")]
-        public string Cipher
-        {
-            get; set;
-        }
+        public required string Cipher { get; set; }
 
         [MessagePack.Key("length")]
         [JsonPropertyName("length")]
@@ -56,9 +50,9 @@ namespace Notesnook.API.Models
         [JsonPropertyName("salt")]
         [BsonElement("salt")]
         [DataMember(Name = "salt")]
-        public string Salt { get; set; }
+        public required string Salt { get; set; }
 
-        public override bool Equals(object obj)
+        public override bool Equals(object? obj)
         {
             if (obj is EncryptedData encryptedData)
             {

Notesnook.API/Models/InboxApiKey.cs

@@ -0,0 +1,69 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Text.Json.Serialization;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using NanoidDotNet;
+
+namespace Notesnook.API.Models
+{
+    public class CreateInboxApiKeyRequest
+    {
+        [JsonPropertyName("name")]
+        public required string Name { get; set; }
+
+        [JsonPropertyName("expiryDate")]
+        public long ExpiryDate { get; set; }
+    }
+
+    public class InboxApiKey
+    {
+        public InboxApiKey()
+        {
+            var random = Nanoid.Generate(size: 64);
+            Key = "nn__" + random;
+        }
+
+        [BsonId]
+        [BsonIgnoreIfDefault]
+        [BsonRepresentation(BsonType.ObjectId)]
+        [JsonIgnore]
+        [MessagePack.IgnoreMember]
+        public string Id { get; set; } = string.Empty;
+
+        [JsonPropertyName("userId")]
+        public required string UserId { get; set; }
+
+        [JsonPropertyName("name")]
+        public required string Name { get; set; }
+
+        [JsonPropertyName("key")]
+        public string Key { get; set; } = string.Empty;
+
+        [JsonPropertyName("dateCreated")]
+        public long DateCreated { get; set; }
+
+        [JsonPropertyName("expiryDate")]
+        public long ExpiryDate { get; set; }
+
+        [JsonPropertyName("lastUsedAt")]
+        public long LastUsedAt { get; set; }
+    }
+}

Notesnook.API/Models/InboxSyncItem.cs

@@ -0,0 +1,85 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.ComponentModel.DataAnnotations;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Notesnook.API.Models
+{
+
+    [MessagePack.MessagePackObject]
+    public class InboxSyncItem
+    {
+        [DataMember(Name = "cipher")]
+        [JsonPropertyName("cipher")]
+        [MessagePack.Key("cipher")]
+        [Required]
+        public string Cipher
+        {
+            get; set;
+        }
+
+        [DataMember(Name = "userId")]
+        [JsonPropertyName("userId")]
+        [MessagePack.Key("userId")]
+        public string? UserId
+        {
+            get; set;
+        }
+
+        [DataMember(Name = "id")]
+        [JsonPropertyName("id")]
+        [MessagePack.Key("id")]
+        public string? ItemId
+        {
+            get; set;
+        }
+
+        [BsonId]
+        [BsonIgnoreIfDefault]
+        [BsonRepresentation(BsonType.ObjectId)]
+        [JsonIgnore]
+        [MessagePack.IgnoreMember]
+        public ObjectId Id
+        {
+            get; set;
+        }
+
+        [JsonPropertyName("v")]
+        [DataMember(Name = "v")]
+        [MessagePack.Key("v")]
+        [Required]
+        public double Version
+        {
+            get; set;
+        }
+
+        [JsonPropertyName("alg")]
+        [DataMember(Name = "alg")]
+        [MessagePack.Key("alg")]
+        [Required]
+        public string Algorithm
+        {
+            get; set;
+        }
+    }
+}

Notesnook.API/Models/Monograph.cs

@@ -29,15 +29,9 @@ namespace Notesnook.API.Models
         [BsonId]
         [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
-        public string Id
-        {
-            get; set;
-        }
+        public required string Id { get; set; }
 
-        public string ItemId
-        {
-            get; set;
-        }
+        public required string ItemId { get; set; }
     }
 
     public class Monograph
@@ -50,47 +44,50 @@ namespace Notesnook.API.Models
         [DataMember(Name = "id")]
         [JsonPropertyName("id")]
         [MessagePack.Key("id")]
-        public string ItemId
-        {
-            get; set;
-        }
+        public string? ItemId { get; set; }
 
         [BsonId]
         [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
         [MessagePack.IgnoreMember]
-        public string Id
-        {
-            get; set;
-        }
+        public string Id { get; set; } = string.Empty;
 
         [JsonPropertyName("title")]
-        public string Title { get; set; }
+        public string? Title { get; set; }
+
+        [JsonPropertyName("slug")]
+        public string? Slug { get; set; }
 
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public string? UserId { get; set; }
 
         [JsonPropertyName("selfDestruct")]
         public bool SelfDestruct { get; set; }
 
         [JsonPropertyName("encryptedContent")]
-        public EncryptedData EncryptedContent { get; set; }
+        public EncryptedData? EncryptedContent { get; set; }
 
         [JsonPropertyName("datePublished")]
         public long DatePublished { get; set; }
 
         [JsonPropertyName("content")]
         [BsonIgnore]
-        public string Content { get; set; }
+        public string? Content { get; set; }
 
         [JsonIgnore]
-        public byte[] CompressedContent { get; set; }
+        public byte[]? CompressedContent { get; set; }
 
         [JsonPropertyName("password")]
-        public EncryptedData Password { get; set; }
+        public EncryptedData? Password { get; set; }
 
         [JsonPropertyName("deleted")]
         public bool Deleted { get; set; }
+
+        [JsonPropertyName("viewCount")]
+        public int ViewCount { get; set; }
+
+        [JsonIgnore]
+        public ContentSanitizationLevel ContentSanitizationLevel { get; set; }
     }
 }

Notesnook.API/Models/MonographContent.cs

@@ -17,15 +17,19 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
-using Streetwriters.Common.Interfaces;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using System.Runtime.Serialization;
 
-namespace Streetwriters.Common.Models
+namespace Notesnook.API.Models
 {
-    public class SubscriptionResponse : Response
+
+    public class MonographContent
     {
-        [JsonPropertyName("subscription")]
-        public ISubscription Subscription { get; set; }
+        [JsonPropertyName("data")]
+        public required string Data { get; set; }
+        [JsonPropertyName("type")]
+        public required string Type { get; set; }
     }
-}
+}

Notesnook.API/Models/MonographMetadata.cs

@@ -17,17 +17,37 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System;
+using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
 
-namespace Streetwriters.Common.Attributes
+namespace Notesnook.API.Models
 {
-    [AttributeUsage(AttributeTargets.Interface, AllowMultiple = false)]
-    public class JsonInterfaceConverterAttribute : JsonConverterAttribute
+    public class MonographMetadata
     {
-        public JsonInterfaceConverterAttribute(Type converterType)
-            : base(converterType)
+        [DataMember(Name = "id")]
+        [JsonPropertyName("id")]
+        [MessagePack.Key("id")]
+        public required string ItemId
         {
+            get; set;
         }
+
+        [JsonPropertyName("title")]
+        public string? Title { get; set; }
+
+        [JsonPropertyName("publishUrl")]
+        public string? PublishUrl { get; set; }
+
+        [JsonPropertyName("selfDestruct")]
+        public bool SelfDestruct { get; set; }
+
+        [JsonPropertyName("datePublished")]
+        public long DatePublished { get; set; }
+
+        [JsonPropertyName("password")]
+        public EncryptedData? Password { get; set; }
+
+        [JsonPropertyName("deleted")]
+        public bool Deleted { get; set; }
     }
 }

Notesnook.API/Models/MultipartUploadMeta.cs

@@ -17,11 +17,13 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
+
 namespace Notesnook.API.Models
 {
     public class MultipartUploadMeta
     {
-        public string UploadId { get; set; }
-        public string[] Parts { get; set; }
+        public string UploadId { get; set; } = string.Empty;
+        public string[] Parts { get; set; } = Array.Empty<string>();
     }
 }

Notesnook.API/Models/PartETagWrapper.cs

@@ -3,5 +3,5 @@
 public class PartETagWrapper
 {
     public int PartNumber { get; set; }
-    public string ETag { get; set; }
+    public string ETag { get; set; } = string.Empty;
 }

Notesnook.API/Models/ResetPasswordForm.cs

@@ -0,0 +1,19 @@
+using System.ComponentModel.DataAnnotations;
+
+namespace Notesnook.API.Models
+{
+    public class ResetPasswordForm
+    {
+        [Required]
+        public required string NewPassword
+        {
+            get; set;
+        }
+
+        [Required]
+        public required UserKeys UserKeys
+        {
+            get; set;
+        }
+    }
+}

Notesnook.API/Models/Responses/SignupResponse.cs

@@ -1,14 +0,0 @@
-using System.Text.Json.Serialization;
-using Streetwriters.Common.Models;
-
-namespace Notesnook.API.Models.Responses
-{
-    public class SignupResponse : Response
-    {
-        [JsonPropertyName("userId")]
-        public string UserId { get; set; }
-
-        [JsonPropertyName("errors")]
-        public string[] Errors { get; set; }
-    }
-}

Notesnook.API/Models/Responses/UserResponse.cs

@@ -1,3 +1,4 @@
+using System.Net.Http;
 using System.Text.Json.Serialization;
 using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Models;
@@ -7,22 +8,35 @@ namespace Notesnook.API.Models.Responses
     public class UserResponse : UserModel, IResponse
     {
         [JsonPropertyName("salt")]
-        public string Salt { get; set; }
+        public string? Salt { get; set; }
 
         [JsonPropertyName("attachmentsKey")]
-        public EncryptedData AttachmentsKey { get; set; }
+        public EncryptedData? AttachmentsKey { get; set; }
 
         [JsonPropertyName("monographPasswordsKey")]
-        public EncryptedData MonographPasswordsKey { get; set; }
+        public EncryptedData? MonographPasswordsKey { get; set; }
+        [JsonPropertyName("dataEncryptionKey")]
+        public EncryptedData? DataEncryptionKey { get; set; }
+
+        [JsonPropertyName("legacyDataEncryptionKey")]
+        public EncryptedData? LegacyDataEncryptionKey { get; set; }
+
+        [JsonPropertyName("inboxKeys")]
+        public InboxKeys? InboxKeys { get; set; }
 
         [JsonPropertyName("subscription")]
-        public ISubscription Subscription { get; set; }
+        public Subscription? Subscription { get; set; }
 
-        [JsonPropertyName("profile")]
-        public EncryptedData Profile { get; set; }
+        [JsonPropertyName("storageUsed")]
+        public long StorageUsed { get; set; }
+
+        [JsonPropertyName("totalStorage")]
+        public long TotalStorage { get; set; }
 
         [JsonIgnore]
         public bool Success { get; set; }
         public int StatusCode { get; set; }
+        [JsonIgnore]
+        public HttpContent? Content { get; set; }
     }
 }

Notesnook.API/Models/S3Options.cs

@@ -21,9 +21,9 @@ namespace Notesnook.API.Models
 {
     public class S3Options
     {
-        public string ServiceUrl { get; set; }
-        public string Region { get; set; }
-        public string AccessKeyId { get; set; }
-        public string SecretAccessKey { get; set; }
+        public string ServiceUrl { get; set; } = string.Empty;
+        public string Region { get; set; } = string.Empty;
+        public string AccessKeyId { get; set; } = string.Empty;
+        public string SecretAccessKey { get; set; } = string.Empty;
     }
 }

Notesnook.API/Models/SyncDevice.cs

@@ -0,0 +1,19 @@
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Notesnook.API.Models
+{
+    public class SyncDevice
+    {
+        [BsonId]
+        public ObjectId Id { get; set; }
+
+        public required string UserId { get; set; }
+        public required string DeviceId { get; set; }
+        public required long LastAccessTime { get; set; }
+        public required bool IsSyncReset { get; set; }
+        public string? AppVersion { get; set; }
+        public string? DatabaseVersion { get; set; }
+    }
+}

Notesnook.API/Models/SyncItem.cs

@@ -44,7 +44,7 @@ namespace Notesnook.API.Models
         [DataMember(Name = "userId")]
         [JsonPropertyName("userId")]
         [MessagePack.Key("userId")]
-        public string UserId
+        public string? UserId
         {
             get; set;
         }
@@ -53,25 +53,19 @@ namespace Notesnook.API.Models
         [DataMember(Name = "iv")]
         [MessagePack.Key("iv")]
         [Required]
-        public string IV
-        {
-            get; set;
-        }
+        public string IV { get; set; } = string.Empty;
 
 
         [JsonPropertyName("cipher")]
         [DataMember(Name = "cipher")]
         [MessagePack.Key("cipher")]
         [Required]
-        public string Cipher
-        {
-            get; set;
-        }
+        public string Cipher { get; set; } = string.Empty;
 
         [DataMember(Name = "id")]
         [JsonPropertyName("id")]
         [MessagePack.Key("id")]
-        public string ItemId
+        public string? ItemId
         {
             get; set;
         }
@@ -104,14 +98,19 @@ namespace Notesnook.API.Models
             get; set;
         }
 
+        [JsonPropertyName("keyVersion")]
+        [DataMember(Name = "keyVersion")]
+        [MessagePack.Key("keyVersion")]
+        public int? KeyVersion
+        {
+            get; set;
+        }
+
         [JsonPropertyName("alg")]
         [DataMember(Name = "alg")]
         [MessagePack.Key("alg")]
         [Required]
-        public string Algorithm
-        {
-            get; set;
-        } = Algorithms.Default;
+        public string Algorithm { get; set; } = string.Empty;
     }
 
     public class SyncItemBsonSerializer : SerializerBase<SyncItem>

Notesnook.API/Models/UserKeys.cs

@@ -19,9 +19,18 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 namespace Notesnook.API.Models
 {
-  public class UserKeys
-  {
-    public EncryptedData AttachmentsKey { get; set; }
-    public EncryptedData MonographPasswordsKey { get; set; }
-  }
+    public class UserKeys
+    {
+        public EncryptedData? AttachmentsKey { get; set; }
+        public EncryptedData? MonographPasswordsKey { get; set; }
+        public InboxKeys? InboxKeys { get; set; }
+        public EncryptedData? DataEncryptionKey { get; set; }
+        public EncryptedData? LegacyDataEncryptionKey { get; set; }
+    }
+
+    public class InboxKeys
+    {
+        public string? Public { get; set; }
+        public EncryptedData? Private { get; set; }
+    }
 }

Notesnook.API/Models/UserSettings.cs

@@ -17,27 +17,51 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Notesnook.API.Interfaces;
 
 namespace Notesnook.API.Models
 {
-    public class UserSettings : IUserSettings
+    public class Limit
+    {
+        private long _value = 0;
+        public long Value
+        {
+            get => _value;
+            set
+            {
+                _value = value;
+                UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+            }
+        }
+        public long UpdatedAt
+        {
+            get;
+            set;
+        }
+    }
+
+    public class UserSettings
     {
         public UserSettings()
         {
-            this.Id = ObjectId.GenerateNewId().ToString();
+            this.Id = ObjectId.GenerateNewId();
         }
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
         public long LastSynced { get; set; }
-        public string Salt { get; set; }
-        public EncryptedData VaultKey { get; set; }
-        public EncryptedData AttachmentsKey { get; set; }
-        public EncryptedData MonographPasswordsKey { get; set; }
+        public required string Salt { get; set; }
+        public EncryptedData? VaultKey { get; set; }
+        public EncryptedData? AttachmentsKey { get; set; }
+        public EncryptedData? MonographPasswordsKey { get; set; }
+        public EncryptedData? DataEncryptionKey { get; set; }
+        public EncryptedData? LegacyDataEncryptionKey { get; set; }
+        public InboxKeys? InboxKeys { get; set; }
+        public Limit? StorageLimit { get; set; }
 
         [BsonId]
         [BsonRepresentation(BsonType.ObjectId)]
-        public string Id { get; set; }
+        public ObjectId Id { get; set; }
     }
 }

Notesnook.API/Notesnook.API.csproj

@@ -1,11 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <StartupObject>Notesnook.API.Program</StartupObject>
+    <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AngleSharp" Version="1.3.0" />
+    <PackageReference Include="AspNetCore.HealthChecks.Aws.S3" Version="9.0.0" />
+    <PackageReference Include="AspNetCore.HealthChecks.Redis" Version="9.0.0" />
     <PackageReference Include="AWSSDK.Core" Version="3.7.304.31" />
     <PackageReference Include="DotNetEnv" Version="2.3.0" />
     <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="6.2.0" />
@@ -14,9 +18,13 @@
     <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="6.0.1-rc2.2" />
     <PackageReference Include="AWSSDK.S3" Version="3.7.310.8" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="6.0.3" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="9.0.13" />
     <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Https" Version="2.2.0" />
+    <PackageReference Include="Nanoid" Version="3.1.0" />
     <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-alpha.2" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
+    <PackageReference Include="Quartz" Version="3.5.0" />
+    <PackageReference Include="Quartz.AspNetCore" Version="3.5.0" />
   </ItemGroup>
 
   <ItemGroup>

Notesnook.API/Program.cs

@@ -17,13 +17,12 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System;
+using System.Net;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Streetwriters.Common;
-using System.Net;
 
 namespace Notesnook.API
 {
@@ -42,6 +41,12 @@ namespace Notesnook.API
 
         public static IHostBuilder CreateHostBuilder(string[] args) =>
             Host.CreateDefaultBuilder(args)
+                .ConfigureLogging(logging =>
+                {
+                    logging.ClearProviders();
+                    logging.AddConsole();
+                    logging.AddSystemdConsole();
+                })
                 .ConfigureWebHostDefaults(webBuilder =>
                 {
                     webBuilder
@@ -50,7 +55,7 @@ namespace Notesnook.API
                     {
                         options.Limits.MaxRequestBodySize = long.MaxValue;
                         options.ListenAnyIP(Servers.NotesnookAPI.Port);
-                        if (Servers.NotesnookAPI.IsSecure)
+                        if (Servers.NotesnookAPI.IsSecure && Servers.NotesnookAPI.SSLCertificate != null)
                         {
                             options.ListenAnyIP(443, listenerOptions =>
                             {

Notesnook.API/Repositories/SyncItemsRepository.cs

@@ -26,6 +26,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using IdentityModel;
 using Microsoft.VisualBasic;
+using Microsoft.Extensions.Logging;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using Notesnook.API.Hubs;
@@ -41,9 +42,11 @@ namespace Notesnook.API.Repositories
     public class SyncItemsRepository : Repository<SyncItem>
     {
         private readonly string collectionName;
-        public SyncItemsRepository(IDbContext dbContext, IMongoCollection<SyncItem> collection) : base(dbContext, collection)
+        private readonly ILogger<SyncItemsRepository> logger;
+        public SyncItemsRepository(IDbContext dbContext, IMongoCollection<SyncItem> collection, ILogger<SyncItemsRepository> logger) : base(dbContext, collection)
         {
             this.collectionName = collection.CollectionNamespace.CollectionName;
+            this.logger = logger;
         }
 
         private readonly List<string> ALGORITHMS = [Algorithms.Default];
@@ -52,24 +55,6 @@ namespace Notesnook.API.Repositories
             return ALGORITHMS.Contains(algorithm);
         }
 
-        public Task<long> CountItemsSyncedAfterAsync(string userId, long timestamp)
-        {
-            var filter = Builders<SyncItem>.Filter.And(Builders<SyncItem>.Filter.Gt("DateSynced", timestamp), Builders<SyncItem>.Filter.Eq("UserId", userId));
-            return Collection.CountDocumentsAsync(filter);
-        }
-        public Task<IAsyncCursor<SyncItem>> FindItemsSyncedAfter(string userId, long timestamp, int batchSize)
-        {
-            var filter = Builders<SyncItem>.Filter.And(Builders<SyncItem>.Filter.Gt("DateSynced", timestamp), Builders<SyncItem>.Filter.Eq("UserId", userId));
-            return Collection.FindAsync(filter, new FindOptions<SyncItem>
-            {
-                BatchSize = batchSize,
-                AllowDiskUse = true,
-                AllowPartialResults = false,
-                NoCursorTimeout = true,
-                Sort = new SortDefinitionBuilder<SyncItem>().Ascending("_id")
-            });
-        }
-
         public Task<IAsyncCursor<SyncItem>> FindItemsById(string userId, IEnumerable<string> ids, bool all, int batchSize)
         {
             var filters = new List<FilterDefinition<SyncItem>>(new[] { Builders<SyncItem>.Filter.Eq("UserId", userId) });
@@ -88,44 +73,10 @@ namespace Notesnook.API.Repositories
         public void DeleteByUserId(string userId)
         {
             var filter = Builders<SyncItem>.Filter.Eq("UserId", userId);
-            var writes = new List<WriteModel<SyncItem>>
-            {
-                new DeleteManyModel<SyncItem>(filter)
-            };
-            dbContext.AddCommand((handle, ct) => Collection.BulkWriteAsync(handle, writes, options: null, ct));
+            dbContext.AddCommand((handle, ct) => Collection.DeleteManyAsync(handle, filter, null, ct));
         }
 
-        public void Upsert(SyncItem item, string userId, long dateSynced)
-        {
-            if (item.Length > 15 * 1024 * 1024)
-            {
-                throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
-            }
-
-            if (!IsValidAlgorithm(item.Algorithm))
-            {
-                throw new Exception($"Invalid alg identifier {item.Algorithm}");
-            }
-
-            // Handle case where the cipher is corrupted.
-            if (!IsBase64String(item.Cipher))
-            {
-                Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
-                throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
-            }
-
-            item.DateSynced = dateSynced;
-            item.UserId = userId;
-
-            var filter = Builders<SyncItem>.Filter.And(
-                Builders<SyncItem>.Filter.Eq("UserId", userId),
-                Builders<SyncItem>.Filter.Eq("ItemId", item.ItemId)
-            );
-
-            dbContext.AddCommand((handle, ct) => Collection.ReplaceOneAsync(handle, filter, item, new ReplaceOptions { IsUpsert = true }, ct));
-        }
-
-        public void UpsertMany(IEnumerable<SyncItem> items, string userId, long dateSynced)
+        public void UpsertMany(IEnumerable<SyncItem> items, string userId)
         {
             var userIdFilter = Builders<SyncItem>.Filter.Eq("UserId", userId);
             var writes = new List<WriteModel<SyncItem>>();
@@ -144,16 +95,20 @@ namespace Notesnook.API.Repositories
                 // Handle case where the cipher is corrupted.
                 if (!IsBase64String(item.Cipher))
                 {
-                    Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
+                    logger.LogError("Corrupted item {ItemId} in collection {CollectionName}. Length: {Length}, Cipher: {Cipher}",
+                        item.ItemId, this.collectionName, item.Length, item.Cipher);
                     throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
                 }
 
+                if (item.ItemId == null)
+                    throw new Exception($"Item does not have an ItemId.");
+
                 var filter = Builders<SyncItem>.Filter.And(
                     userIdFilter,
                     Builders<SyncItem>.Filter.Eq("ItemId", item.ItemId)
                 );
 
-                item.DateSynced = dateSynced;
+                item.DateSynced = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
                 item.UserId = userId;
 
                 writes.Add(new ReplaceOneModel<SyncItem>(filter, item)

Notesnook.API/Services/S3Service.cs

@@ -24,13 +24,15 @@ using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Amazon;
-using Amazon.Runtime;
 using Amazon.S3;
 using Amazon.S3.Model;
-using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
 
 namespace Notesnook.API.Services
 {
@@ -42,9 +44,11 @@ namespace Notesnook.API.Services
 
     public class S3Service : IS3Service
     {
-        private readonly string BUCKET_NAME = Constants.S3_BUCKET_NAME ?? "";
-        private readonly string INTERNAL_BUCKET_NAME = Constants.S3_INTERNAL_BUCKET_NAME ?? "";
-        private AmazonS3Client S3Client { get; }
+        private readonly string BUCKET_NAME = Constants.S3_BUCKET_NAME;
+        private readonly string INTERNAL_BUCKET_NAME = Constants.S3_INTERNAL_BUCKET_NAME ?? Constants.S3_BUCKET_NAME;
+        private readonly S3FailoverHelper S3Client;
+        private ISyncItemsRepositoryAccessor Repositories { get; }
+        private WampServiceAccessor ServiceAccessor { get; }
 
         // When running in a dockerized environment the sync server doesn't have access
         // to the host's S3 Service URL. It can only talk to S3 server via its own internal
@@ -54,60 +58,121 @@ namespace Notesnook.API.Services
         // URLs generated by S3 are host specific. Changing their hostname on the fly causes
         // SignatureDoesNotMatch error.
         // That is why we create 2 separate S3 clients. One for internal traffic and one for external.
-        private AmazonS3Client S3InternalClient { get; }
-        private HttpClient httpClient = new HttpClient();
+        private readonly S3FailoverHelper S3InternalClient;
+        private readonly HttpClient httpClient = new();
 
-        public S3Service()
+        public S3Service(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, WampServiceAccessor wampServiceAccessor, ILogger<S3Service> logger)
         {
-            var config = new AmazonS3Config
-            {
-#if (DEBUG || STAGING)
-                ServiceURL = Servers.S3Server.ToString(),
-#else
-                ServiceURL = Constants.S3_SERVICE_URL,
-                AuthenticationRegion = Constants.S3_REGION,
-#endif
-                ForcePathStyle = true,
-                SignatureMethod = SigningAlgorithm.HmacSHA256,
-                SignatureVersion = "4"
-            };
-#if (DEBUG || STAGING)
-            S3Client = new AmazonS3Client("S3RVER", "S3RVER", config);
-#else
-            S3Client = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, config);
-#endif
+            Repositories = syncItemsRepositoryAccessor;
+            ServiceAccessor = wampServiceAccessor;
+            S3Client = new S3FailoverHelper(
+                S3ClientFactory.CreateS3Clients(
+                    Constants.S3_SERVICE_URL,
+                    Constants.S3_REGION,
+                    Constants.S3_ACCESS_KEY_ID,
+                    Constants.S3_ACCESS_KEY,
+                    forcePathStyle: true
+                ),
+                logger: logger
+            );
 
-            if (!string.IsNullOrEmpty(Constants.S3_INTERNAL_SERVICE_URL))
+            if (!string.IsNullOrEmpty(Constants.S3_INTERNAL_SERVICE_URL) && !string.IsNullOrEmpty(Constants.S3_INTERNAL_BUCKET_NAME))
             {
-                S3InternalClient = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, new AmazonS3Config
-                {
-                    ServiceURL = Constants.S3_INTERNAL_SERVICE_URL,
-                    AuthenticationRegion = Constants.S3_REGION,
-                    ForcePathStyle = true,
-                    SignatureMethod = SigningAlgorithm.HmacSHA256,
-                    SignatureVersion = "4"
-                });
+                S3InternalClient = new S3FailoverHelper(
+                    S3ClientFactory.CreateS3Clients(
+                        Constants.S3_INTERNAL_SERVICE_URL,
+                        Constants.S3_REGION,
+                        Constants.S3_ACCESS_KEY_ID,
+                        Constants.S3_ACCESS_KEY,
+                        forcePathStyle: true
+                    ),
+                    logger: logger
+                );
             }
+            else S3InternalClient = S3Client;
 
             AWSConfigsS3.UseSignatureVersion4 = true;
         }
 
         public async Task DeleteObjectAsync(string userId, string name)
         {
-            var objectName = GetFullObjectName(userId, name);
-            if (objectName == null) throw new Exception("Invalid object name."); ;
+            var objectName = GetFullObjectName(userId, name) ?? throw new Exception("Invalid object name.");
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
+            var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.DeleteObjectAsync(INTERNAL_BUCKET_NAME, objectName), operationName: "DeleteObject", isWriteOperation: true);
 
-            if (!IsSuccessStatusCode(((int)response.HttpStatusCode)))
+            if (!IsSuccessStatusCode((int)response.HttpStatusCode))
                 throw new Exception("Could not delete object.");
         }
 
+        public async Task DeleteObjectsAsync(string userId, string[] names)
+        {
+            var objectsToDelete = new List<KeyVersion>();
+
+            foreach (var name in names)
+            {
+                var objectName = GetFullObjectName(userId, name);
+                if (objectName == null) continue;
+
+                objectsToDelete.Add(new KeyVersion { Key = objectName });
+            }
+
+            if (objectsToDelete.Count == 0)
+            {
+                return;
+            }
+
+            // S3 DeleteObjectsRequest supports max 1000 keys per request
+            var batchSize = 1000;
+            var deleteErrors = new List<DeleteError>();
+            var failedBatches = 0;
+
+            for (int i = 0; i < objectsToDelete.Count; i += batchSize)
+            {
+                var batch = objectsToDelete.Skip(i).Take(batchSize).ToList();
+                var deleteObjectsResponse = await S3InternalClient.ExecuteWithFailoverAsync(
+                    (client) => client.DeleteObjectsAsync(new DeleteObjectsRequest
+                    {
+                        BucketName = INTERNAL_BUCKET_NAME,
+                        Objects = batch,
+                    }),
+                    operationName: "DeleteObjects",
+                    isWriteOperation: true
+                );
+
+                if (!IsSuccessStatusCode((int)deleteObjectsResponse.HttpStatusCode))
+                {
+                    failedBatches++;
+                }
+
+                if (deleteObjectsResponse.DeleteErrors.Count > 0)
+                {
+                    deleteErrors.AddRange(deleteObjectsResponse.DeleteErrors);
+                }
+            }
+
+            if (failedBatches > 0 || deleteErrors.Count > 0)
+            {
+                var errorParts = new List<string>();
+
+                if (failedBatches > 0)
+                {
+                    errorParts.Add($"{failedBatches} batch(es) failed with unsuccessful status code");
+                }
+
+                if (deleteErrors.Count > 0)
+                {
+                    errorParts.Add(string.Join(", ", deleteErrors.Select(e => $"{e.Key}: {e.Message}")));
+                }
+
+                throw new Exception(string.Join("; ", errorParts));
+            }
+        }
+
         public async Task DeleteDirectoryAsync(string userId)
         {
             var request = new ListObjectsV2Request
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = INTERNAL_BUCKET_NAME,
                 Prefix = userId,
             };
 
@@ -115,7 +180,7 @@ namespace Notesnook.API.Services
             var keys = new List<KeyVersion>();
             do
             {
-                response = await GetS3Client(S3ClientMode.INTERNAL).ListObjectsV2Async(request);
+                response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.ListObjectsV2Async(request), operationName: "ListObjectsV2");
                 response.S3Objects.ForEach(obj => keys.Add(new KeyVersion
                 {
                     Key = obj.Key,
@@ -127,12 +192,11 @@ namespace Notesnook.API.Services
 
             if (keys.Count <= 0) return;
 
-            var deleteObjectsResponse = await GetS3Client(S3ClientMode.INTERNAL)
-            .DeleteObjectsAsync(new DeleteObjectsRequest
+            var deleteObjectsResponse = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.DeleteObjectsAsync(new DeleteObjectsRequest
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = INTERNAL_BUCKET_NAME,
                 Objects = keys,
-            });
+            }), operationName: "DeleteObjects", isWriteOperation: true);
 
             if (!IsSuccessStatusCode((int)deleteObjectsResponse.HttpStatusCode))
                 throw new Exception("Could not delete directory.");
@@ -140,44 +204,51 @@ namespace Notesnook.API.Services
 
         public async Task<long> GetObjectSizeAsync(string userId, string name)
         {
-            var url = this.GetPresignedURL(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
+            var url = await this.GetPresignedURLAsync(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
             if (url == null) return 0;
 
             var request = new HttpRequestMessage(HttpMethod.Head, url);
             var response = await httpClient.SendAsync(request);
-            const long MAX_SIZE = 513 * 1024 * 1024; // 512 MB
-            if (!Constants.IS_SELF_HOSTED && response.Content.Headers.ContentLength >= MAX_SIZE)
-            {
-                await this.DeleteObjectAsync(userId, name);
-                throw new Exception("File size exceeds the maximum allowed size.");
-            }
             return response.Content.Headers.ContentLength ?? 0;
         }
 
 
-        public string GetUploadObjectUrl(string userId, string name)
+        public async Task<string?> GetUploadObjectUrlAsync(string userId, string name)
         {
-            var url = this.GetPresignedURL(userId, name, HttpVerb.PUT);
+            return await this.GetPresignedURLAsync(userId, name, HttpVerb.PUT);
+        }
+
+        public async Task<string?> GetInternalUploadObjectUrlAsync(string userId, string name)
+        {
+            return await this.GetPresignedURLAsync(userId, name, HttpVerb.PUT, S3ClientMode.INTERNAL);
+        }
+
+        public async Task<string?> GetDownloadObjectUrlAsync(string userId, string name)
+        {
+            // var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
+            // var subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId);
+
+            // var size = await GetObjectSizeAsync(userId, name);
+            // if (StorageHelper.IsFileSizeExceeded(subscription, size))
+            // {
+            //     var fileSizeLimit = StorageHelper.GetFileSizeLimitForPlan(subscription);
+            //     throw new Exception($"You cannot download files larger than {StorageHelper.FormatBytes(fileSizeLimit)} on this plan.");
+            // }
+
+            var url = await this.GetPresignedURLAsync(userId, name, HttpVerb.GET);
             if (url == null) return null;
             return url;
         }
 
-        public string GetDownloadObjectUrl(string userId, string name)
-        {
-            var url = this.GetPresignedURL(userId, name, HttpVerb.GET);
-            if (url == null) return null;
-            return url;
-        }
-
-        public async Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string uploadId = null)
+        public async Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string? uploadId = null)
         {
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) throw new Exception("Could not initiate multipart upload.");
 
             if (string.IsNullOrEmpty(uploadId))
             {
-                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
-                if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to initiate multipart upload.");
+                var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.InitiateMultipartUploadAsync(INTERNAL_BUCKET_NAME, objectName), operationName: "InitiateMultipartUpload", isWriteOperation: true);
+                if (!IsSuccessStatusCode((int)response.HttpStatusCode)) throw new Exception("Failed to initiate multipart upload.");
 
                 uploadId = response.UploadId;
             }
@@ -185,7 +256,7 @@ namespace Notesnook.API.Services
             var signedUrls = new string[parts];
             for (var i = 0; i < parts; ++i)
             {
-                signedUrls[i] = GetPresignedURLForUploadPart(objectName, uploadId, i + 1);
+                signedUrls[i] = await GetPresignedURLForUploadPartAsync(objectName, uploadId, i + 1);
             }
 
             return new MultipartUploadMeta
@@ -200,49 +271,104 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName, uploadId);
+            var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.AbortMultipartUploadAsync(INTERNAL_BUCKET_NAME, objectName, uploadId), operationName: "AbortMultipartUpload", isWriteOperation: true);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to abort multipart upload.");
         }
 
+        private async Task<long> GetMultipartUploadSizeAsync(string userId, string key, string uploadId)
+        {
+            var objectName = GetFullObjectName(userId, key);
+            var parts = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.ListPartsAsync(INTERNAL_BUCKET_NAME, objectName, uploadId), operationName: "ListParts");
+            long totalSize = 0;
+            foreach (var part in parts.Parts)
+            {
+                totalSize += part.Size;
+            }
+            return totalSize;
+        }
+
         public async Task CompleteMultipartUploadAsync(string userId, CompleteMultipartUploadRequest uploadRequest)
         {
             var objectName = GetFullObjectName(userId, uploadRequest.Key);
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+            if (userSettings == null)
+            {
+                await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
+                throw new Exception("User settings not found.");
+            }
+            userSettings.StorageLimit ??= StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+
+            if (!Constants.IS_SELF_HOSTED)
+            {
+                var subscription = await ServiceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
+
+                long fileSize = await GetMultipartUploadSizeAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
+                if (StorageHelper.IsFileSizeExceeded(subscription, fileSize))
+                {
+                    await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
+                    throw new Exception("Max file size exceeded.");
+                }
+
+                userSettings.StorageLimit.Value += fileSize;
+                if (StorageHelper.IsStorageLimitReached(subscription, userSettings.StorageLimit.Value))
+                {
+                    await this.AbortMultipartUploadAsync(userId, uploadRequest.Key, uploadRequest.UploadId);
+                    throw new Exception("Storage limit reached.");
+                }
+            }
+
             uploadRequest.Key = objectName;
-            uploadRequest.BucketName = GetBucketName(S3ClientMode.INTERNAL);
-            var response = await GetS3Client(S3ClientMode.INTERNAL).CompleteMultipartUploadAsync(uploadRequest);
-            if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to complete multipart upload.");
+            uploadRequest.BucketName = INTERNAL_BUCKET_NAME;
+            var response = await S3InternalClient.ExecuteWithFailoverAsync((client) => client.CompleteMultipartUploadAsync(uploadRequest), operationName: "CompleteMultipartUpload", isWriteOperation: true);
+            if (!IsSuccessStatusCode((int)response.HttpStatusCode)) throw new Exception("Failed to complete multipart upload.");
+
+            if (!Constants.IS_SELF_HOSTED)
+            {
+                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+                await Repositories.UsersSettings.Collection.UpdateOneAsync(
+                    Builders<UserSettings>.Filter.Eq(u => u.UserId, userId),
+                    Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
+                );
+            }
         }
 
-        private string GetPresignedURL(string userId, string name, HttpVerb httpVerb, S3ClientMode mode = S3ClientMode.EXTERNAL)
+        private async Task<string?> GetPresignedURLAsync(string userId, string name, HttpVerb httpVerb, S3ClientMode mode = S3ClientMode.EXTERNAL)
         {
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) return null;
 
-            var client = GetS3Client(mode);
-            var request = new GetPreSignedUrlRequest
+            var client = mode == S3ClientMode.INTERNAL ? S3InternalClient : S3Client;
+            var bucketName = mode == S3ClientMode.INTERNAL ? INTERNAL_BUCKET_NAME : BUCKET_NAME;
+
+            return await client.ExecuteWithFailoverAsync(client =>
             {
-                BucketName = GetBucketName(mode),
-                Expires = System.DateTime.Now.AddHours(1),
-                Verb = httpVerb,
-                Key = objectName,
+                var request = new GetPreSignedUrlRequest
+                {
+                    BucketName = bucketName,
+                    Expires = System.DateTime.Now.AddHours(1),
+                    Verb = httpVerb,
+                    Key = objectName,
 #if (DEBUG || STAGING)
-                Protocol = Protocol.HTTP,
+                    Protocol = Protocol.HTTP,
 #else
-                Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
+                    Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
 #endif
-            };
-            return client.GetPreSignedURL(request);
+                };
+                return client.GetPreSignedURLAsync(request);
+            }, operationName: "GetPreSignedURL");
         }
 
-        private string GetPresignedURLForUploadPart(string objectName, string uploadId, int partNumber, S3ClientMode mode = S3ClientMode.EXTERNAL)
+        private Task<string> GetPresignedURLForUploadPartAsync(string objectName, string uploadId, int partNumber, S3ClientMode mode = S3ClientMode.EXTERNAL)
         {
 
-            var client = GetS3Client(mode);
-            return client.GetPreSignedURL(new GetPreSignedUrlRequest
+            var client = mode == S3ClientMode.INTERNAL ? S3InternalClient : S3Client;
+            var bucketName = mode == S3ClientMode.INTERNAL ? INTERNAL_BUCKET_NAME : BUCKET_NAME;
+
+            return client.ExecuteWithFailoverAsync(c => c.GetPreSignedURLAsync(new GetPreSignedUrlRequest
             {
-                BucketName = GetBucketName(mode),
+                BucketName = bucketName,
                 Expires = System.DateTime.Now.AddHours(1),
                 Verb = HttpVerb.PUT,
                 Key = objectName,
@@ -251,32 +377,20 @@ namespace Notesnook.API.Services
 #if (DEBUG || STAGING)
                 Protocol = Protocol.HTTP,
 #else
-                Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
+                Protocol = c.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
 #endif
-            });
+            }), operationName: "GetPreSignedURL");
         }
 
-        private string GetFullObjectName(string userId, string name)
+        private static string? GetFullObjectName(string userId, string name)
         {
             if (userId == null || !Regex.IsMatch(name, "[0-9a-zA-Z!" + Regex.Escape("-") + "_.*'()]")) return null;
             return $"{userId}/{name}";
         }
 
-        bool IsSuccessStatusCode(int statusCode)
+        static bool IsSuccessStatusCode(int statusCode)
         {
             return ((int)statusCode >= 200) && ((int)statusCode <= 299);
         }
-
-        AmazonS3Client GetS3Client(S3ClientMode mode = S3ClientMode.EXTERNAL)
-        {
-            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return S3InternalClient;
-            return S3Client;
-        }
-
-        string GetBucketName(S3ClientMode mode = S3ClientMode.EXTERNAL)
-        {
-            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return INTERNAL_BUCKET_NAME;
-            return BUCKET_NAME;
-        }
     }
 }

Notesnook.API/Services/SyncDeviceService.cs

@@ -24,210 +24,201 @@ using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Notesnook.API.Interfaces;
+using Notesnook.API.Models;
 
 namespace Notesnook.API.Services
 {
-    public struct SyncDevice(string userId, string deviceId)
+    public readonly record struct ItemKey(string ItemId, string Type)
     {
-        public readonly string DeviceId => deviceId;
-        public readonly string UserId => userId;
-
-        public string UserSyncDirectoryPath = CreateFilePath(userId);
-        public string UserDeviceDirectoryPath = CreateFilePath(userId, deviceId);
-        public string PendingIdsFilePath = CreateFilePath(userId, deviceId, "pending");
-        public string UnsyncedIdsFilePath = CreateFilePath(userId, deviceId, "unsynced");
-        public string ResetSyncFilePath = CreateFilePath(userId, deviceId, "reset-sync");
-
-        public readonly long LastAccessTime
-        {
-            get => long.Parse(GetMetadata("LastAccessTime") ?? "0");
-            set => SetMetadata("LastAccessTime", value.ToString());
-        }
-
-        private static string CreateFilePath(string userId, string? deviceId = null, string? metadataKey = null)
-        {
-            return Path.Join("sync", userId, deviceId, metadataKey);
-        }
-
-        private readonly string? GetMetadata(string metadataKey)
-        {
-            var path = CreateFilePath(userId, deviceId, metadataKey);
-            if (!File.Exists(path)) return null;
-            return File.ReadAllText(path);
-        }
-
-        private readonly void SetMetadata(string metadataKey, string value)
-        {
-            try
-            {
-                var path = CreateFilePath(userId, deviceId, metadataKey);
-                File.WriteAllText(path, value);
-            }
-            catch (DirectoryNotFoundException) { }
-        }
+        public override string ToString() => $"{ItemId}:{Type}";
     }
-
-    public class SyncDeviceService(SyncDevice device)
+    public class SyncDeviceService(ISyncItemsRepositoryAccessor repositories, ILogger<SyncDeviceService> logger)
     {
-        public string[] GetUnsyncedIds()
-        {
-            try
-            {
-                return File.ReadAllLines(device.UnsyncedIdsFilePath);
-            }
-            catch { return []; }
-        }
+        private static FilterDefinition<SyncDevice> DeviceFilter(string userId, string deviceId) =>
+            Builders<SyncDevice>.Filter.Eq(x => x.UserId, userId) &
+            Builders<SyncDevice>.Filter.Eq(x => x.DeviceId, deviceId);
+        private static FilterDefinition<DeviceIdsChunk> DeviceIdsChunkFilter(string userId, string deviceId, string key) =>
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.UserId, userId) &
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.DeviceId, deviceId) &
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.Key, key);
 
-        public string[] GetUnsyncedIds(string deviceId)
-        {
-            try
-            {
-                return File.ReadAllLines(Path.Join(device.UserSyncDirectoryPath, deviceId, "unsynced"));
-            }
-            catch { return []; }
-        }
+        private static FilterDefinition<DeviceIdsChunk> DeviceIdsChunkFilter(string userId, string deviceId) =>
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.UserId, userId) &
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.DeviceId, deviceId);
 
-        public string[] FetchUnsyncedIds()
+        private static FilterDefinition<DeviceIdsChunk> DeviceIdsChunkFilter(string userId) =>
+            Builders<DeviceIdsChunk>.Filter.Eq(x => x.UserId, userId);
+
+        private static FilterDefinition<SyncDevice> UserFilter(string userId) => Builders<SyncDevice>.Filter.Eq(x => x.UserId, userId);
+
+
+        public async Task<HashSet<ItemKey>> GetIdsAsync(string userId, string deviceId, string key)
         {
-            if (IsSyncReset()) return [];
-            try
+            var cursor = await repositories.DeviceIdsChunks.Collection.FindAsync(DeviceIdsChunkFilter(userId, deviceId, key));
+            var result = new HashSet<ItemKey>();
+            while (await cursor.MoveNextAsync())
             {
-                var unsyncedIds = GetUnsyncedIds();
-                lock (device.DeviceId)
+                foreach (var chunk in cursor.Current)
                 {
-                    if (IsSyncPending())
+                    foreach (var id in chunk.Ids)
                     {
-                        unsyncedIds = unsyncedIds.Union(File.ReadAllLines(device.PendingIdsFilePath)).ToArray();
+                        var parts = id.Split(':', 2);
+                        result.Add(new ItemKey(parts[0], parts[1]));
                     }
-
-                    if (unsyncedIds.Length == 0) return [];
-
-                    File.Delete(device.UnsyncedIdsFilePath);
-                    File.WriteAllLines(device.PendingIdsFilePath, unsyncedIds);
                 }
-                return unsyncedIds;
             }
-            catch
+            return result;
+        }
+
+        const int MaxIdsPerChunk = 25_000;
+        public async Task AppendIdsAsync(string userId, string deviceId, string key, IEnumerable<ItemKey> ids)
+        {
+            var filter = DeviceIdsChunkFilter(userId, deviceId, key) & Builders<DeviceIdsChunk>.Filter.Where(x => x.Ids.Length < MaxIdsPerChunk);
+            var chunk = await repositories.DeviceIdsChunks.Collection.Find(filter).FirstOrDefaultAsync();
+
+            if (chunk != null)
             {
-                return [];
+                var update = Builders<DeviceIdsChunk>.Update.AddToSetEach(x => x.Ids, ids.Select(i => i.ToString()));
+                await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).UpdateOneAsync(
+                    Builders<DeviceIdsChunk>.Filter.Eq(x => x.Id, chunk.Id),
+                    update
+                );
             }
-        }
-
-        public void WritePendingIds(IEnumerable<string> ids)
-        {
-            lock (device.DeviceId)
+            else
             {
-                File.WriteAllLines(device.PendingIdsFilePath, ids);
-            }
-        }
-
-        public bool IsSyncReset()
-        {
-            return File.Exists(device.ResetSyncFilePath);
-        }
-        public bool IsSyncReset(string deviceId)
-        {
-            return File.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId, "reset-sync"));
-        }
-
-        public bool IsSyncPending()
-        {
-            return File.Exists(device.PendingIdsFilePath);
-        }
-
-        public bool IsUnsynced()
-        {
-            return File.Exists(device.UnsyncedIdsFilePath);
-        }
-
-        public void Reset()
-        {
-            try
-            {
-                lock (device.UserId)
+                var newChunk = new DeviceIdsChunk
                 {
-                    File.Delete(device.ResetSyncFilePath);
-                    File.Delete(device.PendingIdsFilePath);
-                }
+                    UserId = userId,
+                    DeviceId = deviceId,
+                    Key = key,
+                    Ids = [.. ids.Select(i => i.ToString())]
+                };
+                await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).InsertOneAsync(newChunk);
             }
-            catch (FileNotFoundException) { }
-            catch (DirectoryNotFoundException) { }
+
+            var emptyChunksFilter = DeviceIdsChunkFilter(userId, deviceId, key) & Builders<DeviceIdsChunk>.Filter.Size(x => x.Ids, 0);
+            await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).DeleteManyAsync(emptyChunksFilter);
         }
 
-        public bool IsDeviceRegistered()
+        public async Task WriteIdsAsync(string userId, string deviceId, string key, IEnumerable<ItemKey> ids)
         {
-            return Directory.Exists(device.UserDeviceDirectoryPath);
-        }
-        public bool IsDeviceRegistered(string deviceId)
-        {
-            return Directory.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId));
-        }
-
-        public string[] ListDevices()
-        {
-            return Directory.GetDirectories(device.UserSyncDirectoryPath).Select((path) => path[(path.LastIndexOf(Path.DirectorySeparatorChar) + 1)..]).ToArray();
-        }
-
-        public void ResetDevices()
-        {
-            lock (device.UserId)
+            var writes = new List<WriteModel<DeviceIdsChunk>>
             {
-                if (File.Exists(device.UserSyncDirectoryPath)) File.Delete(device.UserSyncDirectoryPath);
-                Directory.CreateDirectory(device.UserSyncDirectoryPath);
-            }
-        }
-
-        public void AddIdsToOtherDevices(List<string> ids)
-        {
-            device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-            foreach (string id in ListDevices())
+                new DeleteManyModel<DeviceIdsChunk>(DeviceIdsChunkFilter(userId, deviceId, key))
+            };
+            var chunks = ids.Chunk(MaxIdsPerChunk);
+            foreach (var chunk in chunks)
             {
-                if (id == device.DeviceId || IsSyncReset(id)) continue;
-
-                lock (id)
+                var newChunk = new DeviceIdsChunk
                 {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = GetUnsyncedIds(id);
-                    File.WriteAllLines(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds));
-                }
+                    UserId = userId,
+                    DeviceId = deviceId,
+                    Key = key,
+                    Ids = [.. chunk.Select(i => i.ToString())]
+                };
+                writes.Add(new InsertOneModel<DeviceIdsChunk>(newChunk));
             }
+            await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).BulkWriteAsync(writes);
         }
 
-        public void AddIdsToAllDevices(List<string> ids)
+        public async Task<HashSet<ItemKey>> FetchUnsyncedIdsAsync(string userId, string deviceId)
         {
-            foreach (var id in ListDevices())
+            var device = await GetDeviceAsync(userId, deviceId);
+            if (device == null || device.IsSyncReset) return [];
+
+            var unsyncedIds = await GetIdsAsync(userId, deviceId, "unsynced");
+            var pendingIds = await GetIdsAsync(userId, deviceId, "pending");
+
+            unsyncedIds = [.. unsyncedIds, .. pendingIds];
+
+            if (unsyncedIds.Count == 0) return [];
+
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId, deviceId, "unsynced"));
+            await WriteIdsAsync(userId, deviceId, "pending", unsyncedIds);
+
+            return unsyncedIds;
+        }
+
+        public async Task WritePendingIdsAsync(string userId, string deviceId, HashSet<ItemKey> ids)
+        {
+            await WriteIdsAsync(userId, deviceId, "pending", ids);
+        }
+
+        public async Task ResetAsync(string userId, string deviceId)
+        {
+            await repositories.SyncDevices.Collection.UpdateOneAsync(DeviceFilter(userId, deviceId), Builders<SyncDevice>.Update
+                .Set(x => x.IsSyncReset, false));
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId, deviceId, "pending"));
+        }
+
+        public async Task<SyncDevice?> GetDeviceAsync(string userId, string deviceId)
+        {
+            return await repositories.SyncDevices.Collection.Find(DeviceFilter(userId, deviceId)).FirstOrDefaultAsync();
+        }
+
+        public async IAsyncEnumerable<SyncDevice> ListDevicesAsync(string userId)
+        {
+            using var cursor = await repositories.SyncDevices.Collection.FindAsync(UserFilter(userId));
+            while (await cursor.MoveNextAsync())
             {
-                if (IsSyncReset(id)) return;
-                lock (id)
+                foreach (var device in cursor.Current)
                 {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = GetUnsyncedIds(id);
-                    File.WriteAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds));
+                    yield return device;
                 }
             }
         }
 
-        public void RegisterDevice()
+        public async Task ResetDevicesAsync(string userId)
         {
-            lock (device.UserId)
+            await repositories.SyncDevices.Collection.DeleteManyAsync(UserFilter(userId));
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId));
+        }
+
+        public async Task UpdateLastAccessTimeAsync(string userId, string deviceId)
+        {
+            await repositories.SyncDevices.Collection.UpdateOneAsync(DeviceFilter(userId, deviceId), Builders<SyncDevice>.Update
+                .Set(x => x.LastAccessTime, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()));
+        }
+
+        public async Task AddIdsToOtherDevicesAsync(string userId, string deviceId, IEnumerable<ItemKey> ids)
+        {
+            await UpdateLastAccessTimeAsync(userId, deviceId);
+            await foreach (var device in ListDevicesAsync(userId))
             {
-                if (Directory.Exists(device.UserDeviceDirectoryPath))
-                    Directory.Delete(device.UserDeviceDirectoryPath, true);
-                Directory.CreateDirectory(device.UserDeviceDirectoryPath);
-                File.Create(device.ResetSyncFilePath).Close();
-                device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+                if (device.DeviceId == deviceId || device.IsSyncReset) continue;
+                await AppendIdsAsync(userId, device.DeviceId, "unsynced", ids);
             }
         }
 
-        public void UnregisterDevice()
+        public async Task AddIdsToAllDevicesAsync(string userId, IEnumerable<ItemKey> ids)
         {
-            lock (device.UserId)
+            await foreach (var device in ListDevicesAsync(userId))
             {
-                if (!Path.Exists(device.UserDeviceDirectoryPath)) return;
-                Directory.Delete(device.UserDeviceDirectoryPath, true);
+                if (device.IsSyncReset) continue;
+                await AppendIdsAsync(userId, device.DeviceId, "unsynced", ids);
             }
         }
+
+        public async Task<SyncDevice> RegisterDeviceAsync(string userId, string deviceId)
+        {
+            var newDevice = new SyncDevice
+            {
+                UserId = userId,
+                DeviceId = deviceId,
+                LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                IsSyncReset = true
+            };
+            await repositories.SyncDevices.Collection.InsertOneAsync(newDevice);
+            return newDevice;
+        }
+
+        public async Task UnregisterDeviceAsync(string userId, string deviceId)
+        {
+            await repositories.SyncDevices.Collection.DeleteOneAsync(DeviceFilter(userId, deviceId));
+            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(DeviceIdsChunkFilter(userId, deviceId));
+        }
     }
 }

Notesnook.API/Services/UserService.cs

@@ -18,94 +18,89 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.IO;
+using System.Collections.Generic;
 using System.Net.Http;
-using System.Text;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
+using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
 using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Common.Models;
 using Streetwriters.Data.Interfaces;
 
 namespace Notesnook.API.Services
 {
-    public class UserService : IUserService
+    public class UserService(IHttpContextAccessor accessor,
+        ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor,
+        IUnitOfWork unitOfWork, IS3Service s3Service, SyncDeviceService syncDeviceService, WampServiceAccessor serviceAccessor, ILogger<UserService> logger) : IUserService
     {
         private static readonly System.Security.Cryptography.RandomNumberGenerator Rng = System.Security.Cryptography.RandomNumberGenerator.Create();
-        private readonly HttpClient httpClient;
-        private IHttpContextAccessor HttpContextAccessor { get; }
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private IS3Service S3Service { get; set; }
-        private readonly IUnitOfWork unit;
+        private readonly HttpClient httpClient = new();
+        private IHttpContextAccessor HttpContextAccessor { get; } = accessor;
+        private ISyncItemsRepositoryAccessor Repositories { get; } = syncItemsRepositoryAccessor;
+        private IS3Service S3Service { get; set; } = s3Service;
+        private readonly IUnitOfWork unit = unitOfWork;
 
-        public UserService(IHttpContextAccessor accessor,
-        ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor,
-        IUnitOfWork unitOfWork, IS3Service s3Service)
+        public async Task<SignupResponse> CreateUserAsync(SignupForm form)
         {
-            httpClient = new HttpClient();
+            SignupResponse response = await serviceAccessor.UserAccountService.CreateUserAsync(form.ClientId, form.Email, form.Password, HttpContextAccessor.HttpContext?.Request.Headers["User-Agent"].ToString());
 
-            Repositories = syncItemsRepositoryAccessor;
-            HttpContextAccessor = accessor;
-            unit = unitOfWork;
-            S3Service = s3Service;
-        }
-
-        public async Task CreateUserAsync()
-        {
-            SignupResponse response = await httpClient.ForwardAsync<SignupResponse>(this.HttpContextAccessor, $"{Servers.IdentityServer}/signup", HttpMethod.Post);
-            if (!response.Success || (response.Errors != null && response.Errors.Length > 0))
+            if ((response.Errors != null && response.Errors.Length > 0) || response.UserId == null)
             {
-                await Slogger<UserService>.Error(nameof(CreateUserAsync), "Couldn't sign up.", JsonSerializer.Serialize(response));
+                logger.LogError("Failed to sign up user: {Response}", JsonSerializer.Serialize(response));
                 if (response.Errors != null && response.Errors.Length > 0)
                     throw new Exception(string.Join(" ", response.Errors));
-                else throw new Exception("Could not create a new account. Error code: " + response.StatusCode);
+                else throw new Exception("Could not create a new account.");
             }
 
             await Repositories.UsersSettings.InsertAsync(new UserSettings
             {
                 UserId = response.UserId,
+                StorageLimit = new Limit { UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), Value = 0 },
                 LastSynced = 0,
                 Salt = GetSalt()
             });
 
             if (!Constants.IS_SELF_HOSTED)
             {
-                await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.CreateSubscriptionTopic, new CreateSubscriptionMessage
+                await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.CreateSubscriptionV2Topic, new CreateSubscriptionMessageV2
                 {
                     AppId = ApplicationType.NOTESNOOK,
                     Provider = SubscriptionProvider.STREETWRITERS,
-                    Type = SubscriptionType.BASIC,
+                    Status = SubscriptionStatus.ACTIVE,
+                    Plan = SubscriptionPlan.FREE,
                     UserId = response.UserId,
                     StartTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                    Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
                 });
             }
 
-            await Slogger<UserService>.Info(nameof(CreateUserAsync), "New user created.", JsonSerializer.Serialize(response));
+            return response;
         }
 
         public async Task<UserResponse> GetUserAsync(string userId)
         {
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
+            var user = await serviceAccessor.UserAccountService.GetUserAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User not found.");
 
-            var user = await userService.GetUserAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User not found.");
-
-            ISubscription subscription = null;
+            Subscription? subscription = null;
             if (Constants.IS_SELF_HOSTED)
             {
                 subscription = new Subscription
                 {
                     AppId = ApplicationType.NOTESNOOK,
                     Provider = SubscriptionProvider.STREETWRITERS,
-                    Type = SubscriptionType.PREMIUM,
+                    Plan = SubscriptionPlan.BELIEVER,
+                    Status = SubscriptionStatus.ACTIVE,
                     UserId = user.UserId,
                     StartDate = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
                     // this date doesn't matter as the subscription is static.
@@ -114,11 +109,22 @@ namespace Notesnook.API.Services
             }
             else
             {
-                var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
-                subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId);
+                subscription = await serviceAccessor.UserSubscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User subscription not found.");
             }
 
             var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == user.UserId) ?? throw new Exception("User settings not found.");
+
+            // reset user's attachment limit every month
+            var limit = StorageHelper.RolloverStorageLimit(userSettings.StorageLimit);
+            if (userSettings.StorageLimit == null || limit.UpdatedAt != userSettings.StorageLimit?.UpdatedAt)
+            {
+                userSettings.StorageLimit = limit;
+                await Repositories.UsersSettings.Collection.UpdateOneAsync(
+                    Builders<UserSettings>.Filter.Eq(u => u.UserId, user.UserId),
+                    Builders<UserSettings>.Update.Set(u => u.StorageLimit, userSettings.StorageLimit)
+                );
+            }
+
             return new UserResponse
             {
                 UserId = user.UserId,
@@ -129,8 +135,13 @@ namespace Notesnook.API.Services
                 PhoneNumber = user.PhoneNumber,
                 AttachmentsKey = userSettings.AttachmentsKey,
                 MonographPasswordsKey = userSettings.MonographPasswordsKey,
+                DataEncryptionKey = userSettings.DataEncryptionKey,
+                LegacyDataEncryptionKey = userSettings.LegacyDataEncryptionKey,
+                InboxKeys = userSettings.InboxKeys,
                 Salt = userSettings.Salt,
                 Subscription = subscription,
+                StorageUsed = userSettings.StorageLimit.Value,
+                TotalStorage = StorageHelper.GetStorageLimitForPlan(subscription),
                 Success = true,
                 StatusCode = 200
             };
@@ -148,14 +159,42 @@ namespace Notesnook.API.Services
             {
                 userSettings.MonographPasswordsKey = keys.MonographPasswordsKey;
             }
+            if (keys.DataEncryptionKey != null)
+                userSettings.DataEncryptionKey = keys.DataEncryptionKey;
+            if (keys.LegacyDataEncryptionKey != null)
+                userSettings.LegacyDataEncryptionKey = keys.LegacyDataEncryptionKey;
+
+            if (keys.InboxKeys != null)
+            {
+                if (keys.InboxKeys.Public == null || keys.InboxKeys.Private == null)
+                {
+                    userSettings.InboxKeys = null;
+                    await Repositories.InboxApiKey.DeleteManyAsync(t => t.UserId == userId);
+                }
+                else
+                {
+                    userSettings.InboxKeys = keys.InboxKeys;
+                }
+
+                await Repositories.InboxItems.DeleteManyAsync(t => t.UserId == userId);
+                await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+                {
+                    OriginTokenId = null,
+                    UserId = userId,
+                    Message = new Message
+                    {
+                        Type = "inboxUpdated",
+                        Data = JsonSerializer.Serialize(new { reason = "Inbox PGP keys added, updated, or removed." })
+                    }
+                });
+            }
 
             await Repositories.UsersSettings.UpdateAsync(userSettings.Id, userSettings);
         }
 
         public async Task DeleteUserAsync(string userId)
         {
-            new SyncDeviceService(new SyncDevice(userId, userId)).ResetDevices();
-
+            logger.LogInformation("Deleting user {UserId}", userId);
             var cc = new CancellationTokenSource();
 
             Repositories.Notes.DeleteByUserId(userId);
@@ -170,13 +209,17 @@ namespace Notesnook.API.Services
             Repositories.Colors.DeleteByUserId(userId);
             Repositories.Tags.DeleteByUserId(userId);
             Repositories.Vaults.DeleteByUserId(userId);
+            Repositories.InboxItemsHistory.DeleteByUserId(userId);
             Repositories.UsersSettings.Delete((u) => u.UserId == userId);
             Repositories.Monographs.DeleteMany((m) => m.UserId == userId);
+            Repositories.InboxApiKey.DeleteMany((t) => t.UserId == userId);
 
             var result = await unit.Commit();
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "User data deleted", userId, result.ToString());
+            logger.LogInformation("User data deleted for user {UserId}: {Result}", userId, result);
             if (!result) throw new Exception("Could not delete user data.");
 
+            await syncDeviceService.ResetDevicesAsync(userId);
+
             if (!Constants.IS_SELF_HOSTED)
             {
                 await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.DeleteSubscriptionTopic, new DeleteSubscriptionMessage
@@ -189,18 +232,17 @@ namespace Notesnook.API.Services
             await S3Service.DeleteDirectoryAsync(userId);
         }
 
-        public async Task DeleteUserAsync(string userId, string jti, string password)
+        public async Task DeleteUserAsync(string userId, string? jti, string password)
         {
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "Deleting user account", userId);
+            logger.LogInformation("Deleting user account: {UserId}", userId);
 
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
-            await userService.DeleteUserAsync(Clients.Notesnook.Id, userId, password);
+            await serviceAccessor.UserAccountService.DeleteUserAsync(Clients.Notesnook.Id, userId, password);
 
             await DeleteUserAsync(userId);
 
             await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
             {
-                SendToAll = false,
+                SendToAll = jti == null,
                 OriginTokenId = jti,
                 UserId = userId,
                 Message = new Message
@@ -214,7 +256,6 @@ namespace Notesnook.API.Services
 
         public async Task<bool> ResetUserAsync(string userId, bool removeAttachments)
         {
-            new SyncDeviceService(new SyncDevice(userId, userId)).ResetDevices();
 
             var cc = new CancellationTokenSource();
 
@@ -230,14 +271,21 @@ namespace Notesnook.API.Services
             Repositories.Colors.DeleteByUserId(userId);
             Repositories.Tags.DeleteByUserId(userId);
             Repositories.Vaults.DeleteByUserId(userId);
+            Repositories.InboxItemsHistory.DeleteByUserId(userId);
             Repositories.Monographs.DeleteMany((m) => m.UserId == userId);
+            Repositories.InboxApiKey.DeleteMany((t) => t.UserId == userId);
             if (!await unit.Commit()) return false;
 
+            await syncDeviceService.ResetDevicesAsync(userId);
+
             var userSettings = await Repositories.UsersSettings.FindOneAsync((s) => s.UserId == userId);
 
             userSettings.AttachmentsKey = null;
             userSettings.MonographPasswordsKey = null;
+            userSettings.DataEncryptionKey = null;
+            userSettings.LegacyDataEncryptionKey = null;
             userSettings.VaultKey = null;
+            userSettings.InboxKeys = null;
             userSettings.LastSynced = 0;
 
             await Repositories.UsersSettings.UpsertAsync(userSettings, (s) => s.UserId == userId);

Notesnook.API/Startup.cs

@@ -24,6 +24,8 @@ using System.Security.Claims;
 using System.Text;
 using System.Text.Encodings.Web;
 using System.Threading.Tasks;
+using Amazon.Runtime;
+using StackExchange.Redis;
 using IdentityModel.AspNetCore.OAuth2Introspection;
 using Microsoft.AspNetCore.Authentication;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
@@ -39,6 +41,7 @@ using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -48,15 +51,19 @@ using Notesnook.API.Authorization;
 using Notesnook.API.Extensions;
 using Notesnook.API.Hubs;
 using Notesnook.API.Interfaces;
+using Notesnook.API.Jobs;
 using Notesnook.API.Models;
 using Notesnook.API.Repositories;
 using Notesnook.API.Services;
 using OpenTelemetry.Metrics;
 using OpenTelemetry.Resources;
+using Quartz;
 using Streetwriters.Common;
 using Streetwriters.Common.Extensions;
+using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Common.Models;
+using Streetwriters.Common.Services;
 using Streetwriters.Data;
 using Streetwriters.Data.DbContexts;
 using Streetwriters.Data.Interfaces;
@@ -108,16 +115,15 @@ namespace Notesnook.API
                     policy.RequireAuthenticatedUser();
                     policy.Requirements.Add(new SyncRequirement());
                 });
-                options.AddPolicy("Pro", policy =>
+
+                options.AddPolicy(InboxApiKeyAuthenticationDefaults.AuthenticationScheme, policy =>
                 {
-                    policy.AuthenticationSchemes.Add("introspection");
+                    policy.AuthenticationSchemes.Add(InboxApiKeyAuthenticationDefaults.AuthenticationScheme);
                     policy.RequireAuthenticatedUser();
-                    policy.Requirements.Add(new SyncRequirement());
-                    policy.Requirements.Add(new ProUserRequirement());
                 });
 
-                options.DefaultPolicy = options.GetPolicy("Notesnook");
-            }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>(); ;
+                options.DefaultPolicy = options.GetPolicy("Notesnook") ?? throw new Exception("Notesnook policy not found");
+            }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>();
 
             services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
             .AddOAuth2Introspection("introspection", options =>
@@ -135,20 +141,24 @@ namespace Notesnook.API
 
                 options.Events.OnTokenValidated = (context) =>
                 {
-                    if (long.TryParse(context.Principal.FindFirst("exp")?.Value, out long expiryTime))
+                    if (long.TryParse(context.Principal?.FindFirst("exp")?.Value, out long expiryTime))
                     {
                         context.Properties.ExpiresUtc = DateTimeOffset.FromUnixTimeSeconds(expiryTime);
                     }
                     context.Properties.AllowRefresh = true;
                     context.Properties.IsPersistent = true;
-                    context.HttpContext.User = context.Principal;
+                    context.HttpContext.User = context.Principal ?? throw new Exception("No principal found in token.");
                     return Task.CompletedTask;
                 };
                 options.CacheKeyGenerator = (options, token) => (token + ":" + "reference_token").Sha256();
                 options.SaveToken = true;
                 options.EnableCaching = true;
                 options.CacheDuration = TimeSpan.FromMinutes(30);
-            });
+            })
+            .AddScheme<InboxApiKeyAuthenticationSchemeOptions, InboxApiKeyAuthenticationHandler>(
+                InboxApiKeyAuthenticationDefaults.AuthenticationScheme,
+                options => { }
+            );
 
             // Serializer.RegisterSerializer(new SyncItemBsonSerializer());
             if (!BsonClassMap.IsClassMapRegistered(typeof(UserSettings)))
@@ -160,12 +170,19 @@ namespace Notesnook.API
             if (!BsonClassMap.IsClassMapRegistered(typeof(CallToAction)))
                 BsonClassMap.RegisterClassMap<CallToAction>();
 
+            if (!BsonClassMap.IsClassMapRegistered(typeof(SyncDevice)))
+                BsonClassMap.RegisterClassMap<SyncDevice>();
+
             services.AddScoped<IDbContext, MongoDbContext>();
             services.AddScoped<IUnitOfWork, UnitOfWork>();
 
             services.AddRepository<UserSettings>("user_settings", "notesnook")
                     .AddRepository<Monograph>("monographs", "notesnook")
-                    .AddRepository<Announcement>("announcements", "notesnook");
+                    .AddRepository<Announcement>("announcements", "notesnook")
+                    .AddRepository<DeviceIdsChunk>(Collections.DeviceIdsChunksKey, "notesnook")
+                    .AddRepository<SyncDevice>(Collections.SyncDevicesKey, "notesnook")
+                    .AddRepository<InboxApiKey>(Collections.InboxApiKeysKey, "notesnook")
+                    .AddRepository<InboxSyncItem>(Collections.InboxItemsKey, "notesnook");
 
             services.AddMongoCollection(Collections.SettingsKey)
                     .AddMongoCollection(Collections.AttachmentsKey)
@@ -178,22 +195,47 @@ namespace Notesnook.API
                     .AddMongoCollection(Collections.ShortcutsKey)
                     .AddMongoCollection(Collections.TagsKey)
                     .AddMongoCollection(Collections.ColorsKey)
-                    .AddMongoCollection(Collections.VaultsKey);
+                    .AddMongoCollection(Collections.VaultsKey)
+                    .AddMongoCollection(Collections.InboxItemsKey)
+                    .AddMongoCollection(Collections.InboxApiKeysKey)
+                    .AddMongoCollection(Collections.InboxItemsHistoryKey);
 
             services.AddScoped<ISyncItemsRepositoryAccessor, SyncItemsRepositoryAccessor>();
+            services.AddScoped<SyncDeviceService>();
             services.AddScoped<IUserService, UserService>();
             services.AddScoped<IS3Service, S3Service>();
+            services.AddScoped<IURLAnalyzer, URLAnalyzer>();
+
+            services.AddWampServiceAccessor(Servers.NotesnookAPI);
 
             services.AddControllers();
 
-            services.AddHealthChecks(); // .AddMongoDb(dbSettings.ConnectionString, dbSettings.DatabaseName, "database-check");
-            services.AddSignalR((hub) =>
+            services.AddHealthChecks();
+
+            var signalR = services.AddSignalR((hub) =>
             {
                 hub.MaximumReceiveMessageSize = 100 * 1024 * 1024;
+                hub.KeepAliveInterval = TimeSpan.FromSeconds(15);
                 hub.ClientTimeoutInterval = TimeSpan.FromMinutes(10);
                 hub.EnableDetailedErrors = true;
             }).AddMessagePackProtocol().AddJsonProtocol();
 
+            if (!string.IsNullOrEmpty(Constants.SIGNALR_REDIS_CONNECTION_STRING))
+            {
+                services.AddHealthChecks()
+                        .AddRedis(Constants.SIGNALR_REDIS_CONNECTION_STRING, tags: ["ready"]);
+                signalR.AddStackExchangeRedis(options =>
+                {
+                    options.Configuration = ConfigurationOptions.Parse(Constants.SIGNALR_REDIS_CONNECTION_STRING);
+                    options.Configuration.AbortOnConnectFail = false;
+                    options.Configuration.ConnectRetry = 5;
+                    options.Configuration.ReconnectRetryPolicy = new ExponentialRetry(5000, 30000);
+                    options.Configuration.KeepAlive = 60;
+                    options.Configuration.ConnectTimeout = 5000;
+                    options.Configuration.SyncTimeout = 5000;
+                });
+            }
+
             services.AddResponseCompression(options =>
             {
                 options.EnableForHttps = true;
@@ -216,22 +258,40 @@ namespace Notesnook.API
                     .WithMetrics((builder) => builder
                             .AddMeter("Notesnook.API.Metrics.Sync")
                             .AddPrometheusExporter());
+
+            services.AddQuartzHostedService(q =>
+            {
+                q.WaitForJobsToComplete = false;
+                q.AwaitApplicationStarted = true;
+                q.StartDelay = TimeSpan.FromMinutes(1);
+            }).AddQuartz(q =>
+            {
+                q.UseMicrosoftDependencyInjectionJobFactory();
+
+                var jobKey = new JobKey("DeviceCleanupJob");
+                q.AddJob<DeviceCleanupJob>(opts => opts.WithIdentity(jobKey));
+                q.AddTrigger(opts => opts
+                    .ForJob(jobKey)
+                    .WithIdentity("DeviceCleanup-trigger")
+                    // first of every month
+                    .WithCronSchedule("0 0 0 1 * ? *"));
+            });
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
         public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         {
-            if (!env.IsDevelopment())
-            {
-                app.UseForwardedHeaders(new ForwardedHeadersOptions
-                {
-                    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
-                });
-            }
+            app.UseForwardedHeadersWithKnownProxies(env);
 
             app.UseOpenTelemetryPrometheusScrapingEndpoint((context) => context.Request.Path == "/metrics" && context.Connection.LocalPort == 5067);
             app.UseResponseCompression();
 
+            app.UseWebSockets(new Microsoft.AspNetCore.Builder.WebSocketOptions
+            {
+                KeepAliveInterval = TimeSpan.FromSeconds(30),
+                KeepAliveTimeout = TimeSpan.FromSeconds(60),
+            });
+
             app.UseCors("notesnook");
             app.UseVersion(Servers.NotesnookAPI);
 
@@ -257,14 +317,8 @@ namespace Notesnook.API
 
             app.UseEndpoints(endpoints =>
             {
-                endpoints.MapPrometheusScrapingEndpoint();
                 endpoints.MapControllers();
                 endpoints.MapHealthChecks("/health");
-                endpoints.MapHub<SyncHub>("/hubs/sync", options =>
-                {
-                    options.CloseOnAuthenticationExpiration = false;
-                    options.Transports = HttpTransportType.WebSockets;
-                });
                 endpoints.MapHub<SyncV2Hub>("/hubs/sync/v2", options =>
                 {
                     options.CloseOnAuthenticationExpiration = false;
@@ -278,7 +332,7 @@ namespace Notesnook.API
     {
         public static IServiceCollection AddMongoCollection(this IServiceCollection services, string collectionName, string database = "notesnook")
         {
-            services.AddKeyedSingleton(collectionName, (provider, key) => MongoDbContext.GetMongoCollection<SyncItem>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
+            services.AddKeyedSingleton(collectionName, (provider, key) => MongoDbContext.GetMongoCollection<SyncItem>(provider.GetRequiredService<MongoDB.Driver.IMongoClient>(), database, collectionName));
             return services;
         }
     }

Notesnook.API/appsettings.Development.json

@@ -1,7 +1,7 @@
 {
   "Logging": {
     "LogLevel": {
-      "Default": "Information",
+      "Default": "Debug",
       "Microsoft": "Warning",
       "Microsoft.Hosting.Lifetime": "Information",
       "Microsoft.AspNetCore.SignalR": "Trace",

Notesnook.Inbox.API/.env.example

@@ -0,0 +1,2 @@
+PORT=5181
+NOTESNOOK_API_SERVER_URL=http://localhost:5264/

Notesnook.Inbox.API/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+dist/
+.env

Notesnook.Inbox.API/Dockerfile

@@ -0,0 +1,19 @@
+FROM oven/bun:1.3.5-slim
+
+RUN mkdir -p /home/bun/app && chown -R bun:bun /home/bun/app
+
+WORKDIR /home/bun/app
+
+USER bun
+
+COPY --chown=bun:bun package.json bun.lock .
+
+RUN bun install --frozen-lockfile
+
+COPY --chown=bun:bun . .
+
+RUN bun run build
+
+EXPOSE 5181
+
+CMD ["bun", "run", "start"]

Notesnook.Inbox.API/README.md

@@ -0,0 +1,68 @@
+# Notesnook Inbox API
+
+## Running locally
+
+### Requirements
+
+- Bun (v1.3.0 or higher)
+
+### Commands
+
+- `bun install` - Install dependencies
+- `bun run dev` - Start the development server
+- `bun run build` - Build the project for production
+- `bun run start` - Start the production server
+
+## Self-hosting
+
+The easiest way to self-host is with Docker or Docker Compose.
+
+Prerequisites:
+
+- `docker` (Engine) installed
+- `docker-compose` (optional, for multi-service setups)
+
+Build and run with Docker:
+
+```bash
+# build the image from the current folder
+docker build -t notesnook-inbox-api .
+
+# run the container (example)
+docker run --rm -p 3000:3000 \
+	-e PORT=3000 \
+	-e NOTESNOOK_API_SERVER_URL="https://api.notesnook.com" \
+	notesnook-inbox-api
+```
+
+Docker Compose (example):
+
+```yaml
+services:
+	inbox-api:
+		image: notesnook-inbox-api
+		build: .
+		ports:
+			- "3000:3000"
+		environment:
+			PORT: 3000
+			NOTESNOOK_API_SERVER_URL: "https://api.notesnook.com"
+		restart: unless-stopped
+```
+
+Environment variables:
+
+- `PORT` — port the service listens on (default: `5181`)
+- `NOTESNOOK_API_SERVER_URL` — base URL of the Notesnook API used to fetch public inbox keys
+
+_If you prefer running without Docker, use `bun install` and `bun run start` with the environment variables set._
+
+## Writing from scratch
+
+The inbox API server is pretty simple to write from scratch in any programming language and/or framework. There's only one endpoint that needs to be implemented, which does these three steps:
+
+1. Fetch the user's public inbox API key from the Notesnook API.
+2. Encrypt the payload using `openpgp` or any other `openpgp` compatible library.
+3. Post the encrypted payload to the Notesnook API.
+
+You can refer to the [source code](./src/index.ts) for implementation details.

Notesnook.Inbox.API/bun.lock

@@ -0,0 +1,195 @@
+{
+  "lockfileVersion": 1,
+  "workspaces": {
+    "": {
+      "name": "notesnook-inbox-api",
+      "dependencies": {
+        "express": "^5.1.0",
+        "express-rate-limit": "^8.1.0",
+        "openpgp": "^6.2.2",
+        "zod": "^4.1.9",
+      },
+      "devDependencies": {
+        "@types/express": "^5.0.3",
+        "@types/libsodium-wrappers-sumo": "^0.7.8",
+        "@types/node": "^24.5.2",
+        "typescript": "^5.9.2",
+      },
+    },
+  },
+  "packages": {
+    "@types/body-parser": ["@types/body-parser@1.19.6", "", { "dependencies": { "@types/connect": "*", "@types/node": "*" } }, "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g=="],
+
+    "@types/connect": ["@types/connect@3.4.38", "", { "dependencies": { "@types/node": "*" } }, "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug=="],
+
+    "@types/express": ["@types/express@5.0.3", "", { "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", "@types/serve-static": "*" } }, "sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw=="],
+
+    "@types/express-serve-static-core": ["@types/express-serve-static-core@5.0.7", "", { "dependencies": { "@types/node": "*", "@types/qs": "*", "@types/range-parser": "*", "@types/send": "*" } }, "sha512-R+33OsgWw7rOhD1emjU7dzCDHucJrgJXMA5PYCzJxVil0dsyx5iBEPHqpPfiKNJQb7lZ1vxwoLR4Z87bBUpeGQ=="],
+
+    "@types/http-errors": ["@types/http-errors@2.0.5", "", {}, "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg=="],
+
+    "@types/libsodium-wrappers": ["@types/libsodium-wrappers@0.7.14", "", {}, "sha512-5Kv68fXuXK0iDuUir1WPGw2R9fOZUlYlSAa0ztMcL0s0BfIDTqg9GXz8K30VJpPP3sxWhbolnQma2x+/TfkzDQ=="],
+
+    "@types/libsodium-wrappers-sumo": ["@types/libsodium-wrappers-sumo@0.7.8", "", { "dependencies": { "@types/libsodium-wrappers": "*" } }, "sha512-N2+df4MB/A+W0RAcTw7A5oxKgzD+Vh6Ye7lfjWIi5SdTzVLfHPzxUjhwPqHLO5Ev9fv/+VHl+sUaUuTg4fUPqw=="],
+
+    "@types/mime": ["@types/mime@1.3.5", "", {}, "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w=="],
+
+    "@types/node": ["@types/node@24.5.2", "", { "dependencies": { "undici-types": "~7.12.0" } }, "sha512-FYxk1I7wPv3K2XBaoyH2cTnocQEu8AOZ60hPbsyukMPLv5/5qr7V1i8PLHdl6Zf87I+xZXFvPCXYjiTFq+YSDQ=="],
+
+    "@types/qs": ["@types/qs@6.14.0", "", {}, "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ=="],
+
+    "@types/range-parser": ["@types/range-parser@1.2.7", "", {}, "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ=="],
+
+    "@types/send": ["@types/send@0.17.5", "", { "dependencies": { "@types/mime": "^1", "@types/node": "*" } }, "sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w=="],
+
+    "@types/serve-static": ["@types/serve-static@1.15.8", "", { "dependencies": { "@types/http-errors": "*", "@types/node": "*", "@types/send": "*" } }, "sha512-roei0UY3LhpOJvjbIP6ZZFngyLKl5dskOtDhxY5THRSpO+ZI+nzJ+m5yUMzGrp89YRa7lvknKkMYjqQFGwA7Sg=="],
+
+    "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
+
+    "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="],
+
+    "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="],
+
+    "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
+
+    "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
+
+    "content-disposition": ["content-disposition@1.0.0", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg=="],
+
+    "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="],
+
+    "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
+
+    "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
+
+    "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
+
+    "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
+
+    "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
+
+    "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="],
+
+    "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
+
+    "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
+
+    "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
+
+    "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
+
+    "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
+
+    "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
+
+    "express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="],
+
+    "express-rate-limit": ["express-rate-limit@8.1.0", "", { "dependencies": { "ip-address": "10.0.1" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-4nLnATuKupnmwqiJc27b4dCFmB/T60ExgmtDD7waf4LdrbJ8CPZzZRHYErDYNhoz+ql8fUdYwM/opf90PoPAQA=="],
+
+    "finalhandler": ["finalhandler@2.1.0", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q=="],
+
+    "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
+
+    "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
+
+    "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
+
+    "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
+
+    "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
+
+    "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
+
+    "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
+
+    "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
+
+    "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
+
+    "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
+
+    "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
+
+    "ip-address": ["ip-address@10.0.1", "", {}, "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA=="],
+
+    "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
+
+    "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],
+
+    "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
+
+    "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
+
+    "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="],
+
+    "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="],
+
+    "mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="],
+
+    "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
+
+    "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
+
+    "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
+
+    "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="],
+
+    "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
+
+    "openpgp": ["openpgp@6.2.2", "", {}, "sha512-P/dyEqQ3gfwOCo+xsqffzXjmUhGn4AZTOJ1LCcN21S23vAk+EAvMJOQTsb/C8krL6GjOSBxqGYckhik7+hneNw=="],
+
+    "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
+
+    "path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="],
+
+    "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="],
+
+    "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="],
+
+    "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
+
+    "raw-body": ["raw-body@3.0.1", "", { "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", "iconv-lite": "0.7.0", "unpipe": "1.0.0" } }, "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA=="],
+
+    "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="],
+
+    "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
+
+    "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
+
+    "send": ["send@1.2.0", "", { "dependencies": { "debug": "^4.3.5", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.0", "mime-types": "^3.0.1", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.1" } }, "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw=="],
+
+    "serve-static": ["serve-static@2.2.0", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ=="],
+
+    "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
+
+    "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
+
+    "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="],
+
+    "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="],
+
+    "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
+
+    "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="],
+
+    "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
+
+    "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
+
+    "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="],
+
+    "undici-types": ["undici-types@7.12.0", "", {}, "sha512-goOacqME2GYyOZZfb5Lgtu+1IDmAlAEu5xnD3+xTzS10hT0vzpf0SPjkXwAw9Jm+4n/mQGDP3LO8CPbYROeBfQ=="],
+
+    "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
+
+    "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
+
+    "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
+
+    "zod": ["zod@4.1.9", "", {}, "sha512-HI32jTq0AUAC125z30E8bQNz0RQ+9Uc+4J7V97gLYjZVKRjeydPgGt6dvQzFrav7MYOUGFqqOGiHpA/fdbd0cQ=="],
+
+    "http-errors/statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="],
+
+    "raw-body/iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="],
+  }
+}

Notesnook.Inbox.API/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "notesnook-inbox-api",
+  "version": "1.0.0",
+  "description": "Notesnook Inbox API server",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "bun build src/index.ts --outdir dist --target bun",
+    "start": "bun run dist/index.js",
+    "dev": "bun --watch src/index.ts"
+  },
+  "keywords": [
+    "notesnook",
+    "inbox",
+    "api"
+  ],
+  "license": "GPL-3.0-or-later",
+  "author": {
+    "name": "Streetwriters (Private) Limited",
+    "email": "support@streetwriters.co",
+    "url": "https://streetwriters.co"
+  },
+  "dependencies": {
+    "express": "^5.1.0",
+    "express-rate-limit": "^8.1.0",
+    "openpgp": "^6.2.2",
+    "zod": "^4.1.9"
+  },
+  "devDependencies": {
+    "@types/libsodium-wrappers-sumo": "^0.7.8",
+    "@types/express": "^5.0.3",
+    "@types/node": "^24.5.2",
+    "typescript": "^5.9.2"
+  }
+}

Notesnook.Inbox.API/scripts/test.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+GNUPGHOME=$(mktemp -d)
+
+curl -s http://localhost:5264/inbox/public-encryption-key -H "Authorization: $API_KEY" | jq -r .key > "$GNUPGHOME"/pubkey.asc && gpg --batch --homedir "$GNUPGHOME" --import "$GNUPGHOME"/pubkey.asc >/dev/null 2>&1 && KEYID=$(gpg --homedir "$GNUPGHOME" --list-keys --with-colons | awk -F: '/^pub:/ {print $5; exit}') && printf '%s' '{"title":"Test title CLIE S","type":"note","source":"cli","version":1}' | gpg --batch --homedir "$GNUPGHOME" --trust-model always --armor --encrypt -r "$KEYID" | jq -Rs --arg alg "pgp-aes256" '{v:1, cipher:., alg:$alg}' | curl -s -X POST http://localhost:5264/inbox/items -H "Content-Type: application/json" -H "Authorization: $API_KEY" -d @- && rm -rf "$GNUPGHOME"

Notesnook.Inbox.API/scripts/test.ts

@@ -0,0 +1,18 @@
+const response = await fetch("http://localhost:5181/inbox", {
+  method: "POST",
+  headers: {
+    Authorization: process.env.API_KEY,
+    "Content-Type": "application/json",
+  },
+  body: JSON.stringify({
+    title: "This is test note 4",
+    type: "note",
+    source: "script",
+    version: 1,
+    content: {
+      type: "html",
+      data: "<p>This is test note content 3</p>",
+    },
+  }),
+});
+console.log(await response.text());

Notesnook.Inbox.API/src/index.ts

@@ -0,0 +1,167 @@
+import express from "express";
+import { z } from "zod";
+import { rateLimit } from "express-rate-limit";
+import * as openpgp from "openpgp";
+
+const NOTESNOOK_API_SERVER_URL = process.env.NOTESNOOK_API_SERVER_URL;
+if (!NOTESNOOK_API_SERVER_URL) {
+  throw new Error("NOTESNOOK_API_SERVER_URL is not defined");
+}
+
+const RawInboxItemSchema = z.object({
+  title: z.string().min(1, "Title is required"),
+  pinned: z.boolean().optional(),
+  favorite: z.boolean().optional(),
+  readonly: z.boolean().optional(),
+  archived: z.boolean().optional(),
+  notebookIds: z.array(z.string()).optional(),
+  tagIds: z.array(z.string()).optional(),
+  type: z.enum(["note"]),
+  source: z.string().min(1, "Source is required"),
+  version: z.literal(1),
+  content: z
+    .object({
+      type: z.enum(["html"]),
+      data: z.string(),
+    })
+    .optional(),
+});
+
+interface EncryptedInboxItem {
+  v: 1;
+  cipher: string;
+  alg: string;
+}
+
+/**
+ * Encrypts raw data using OpenPGP with the recipient's public key
+ *
+ * @param {string} rawData - The plaintext data to encrypt
+ * @param {string} rawPublicKey - The recipient's OpenPGP public key
+ */
+async function encrypt(
+  rawData: string,
+  rawPublicKey: string,
+): Promise<EncryptedInboxItem> {
+  const publicKey = await openpgp.readKey({ armoredKey: rawPublicKey });
+  const message = await openpgp.createMessage({ text: rawData });
+  const encrypted = await openpgp.encrypt({
+    message,
+    encryptionKeys: publicKey,
+  });
+  return {
+    v: 1,
+    cipher: encrypted,
+    alg: "pgp-aes256",
+  };
+}
+
+async function getInboxPublicEncryptionKey(
+  apiKey: string,
+): Promise<{ status: "unauthorized" } | { status: "ok"; key: string | null }> {
+  const response = await fetch(
+    `${NOTESNOOK_API_SERVER_URL}/inbox/public-encryption-key`,
+    {
+      headers: {
+        Authorization: apiKey,
+      },
+    },
+  );
+  if (response.status === 401) {
+    return { status: "unauthorized" };
+  }
+  if (!response.ok) {
+    throw new Error(
+      `failed to fetch inbox public encryption key: ${await response.text()}`,
+    );
+  }
+
+  const data = (await response.json()) as unknown as any;
+  return { status: "ok", key: (data?.key as string) || null };
+}
+
+async function postEncryptedInboxItem(
+  apiKey: string,
+  item: EncryptedInboxItem,
+) {
+  const response = await fetch(`${NOTESNOOK_API_SERVER_URL}/inbox/items`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: apiKey,
+    },
+    body: JSON.stringify({ ...item }),
+  });
+  if (!response.ok) {
+    throw new Error(`failed to post inbox item: ${await response.text()}`);
+  }
+}
+
+const app = express();
+app.use(express.json({ limit: "10mb" }));
+app.use(
+  rateLimit({
+    windowMs: 1 * 60 * 1000, // 1 minute
+    limit: 60,
+  }),
+);
+app.get("/health", (_, res) => {
+  return res.status(200).json({ status: "ok" });
+});
+app.post("/", async (req, res) => {
+  try {
+    const apiKey = req.headers["authorization"];
+    if (!apiKey) {
+      return res.status(401).json({ error: "unauthorized" });
+    }
+
+    const encryptionKeyResult = await getInboxPublicEncryptionKey(apiKey);
+    if (encryptionKeyResult.status === "unauthorized") {
+      return res.status(401).json({ error: "unauthorized" });
+    }
+    if (!encryptionKeyResult.key) {
+      return res.status(404).json({ error: "inbox public key not found" });
+    }
+    const inboxPublicKey = encryptionKeyResult.key;
+    console.log("[info] fetched inbox public key");
+
+    const validationResult = RawInboxItemSchema.safeParse(req.body);
+    if (!validationResult.success) {
+      return res.status(400).json({
+        error: "invalid item",
+        details: validationResult.error.issues,
+      });
+    }
+
+    const encryptedItem = await encrypt(
+      JSON.stringify(validationResult.data),
+      inboxPublicKey,
+    );
+    console.log("[info] encrypted item");
+
+    await postEncryptedInboxItem(apiKey, encryptedItem);
+    console.log("[info] posted encrypted inbox item successfully");
+
+    return res.status(200).json({ success: true });
+  } catch (error) {
+    if (error instanceof Error) {
+      console.log("[error]", error.message);
+      return res
+        .status(500)
+        .json({ error: "internal server error", description: error.message });
+    } else {
+      console.log("[error] unknown error occured:", error);
+      return res.status(500).json({
+        error: "internal server error",
+        description: `unknown error occured: ${error}`,
+      });
+    }
+  }
+});
+
+const PORT = Number(process.env.PORT || "5181");
+app.listen(PORT, () => {
+  console.log(`📫 notesnook inbox api server running on port ${PORT}`);
+});
+
+export default app;

Notesnook.Inbox.API/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "ESNext",
+    "lib": ["ES2020"],
+    "outDir": "./dist",
+    "strict": true,
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

README.md

@@ -55,17 +55,26 @@ dotnet run --project Streetwriters.Identity/Streetwriters.Identity.csproj
 
 The sync server can easily be started using Docker.
 
+Download `docker-compose.yml` and [`.env`](.env):
+
 ```bash
 wget https://raw.githubusercontent.com/streetwriters/notesnook-sync-server/master/docker-compose.yml
+wget https://raw.githubusercontent.com/streetwriters/notesnook-sync-server/master/.env
 ```
 
-And then use Docker Compose to start the servers:
+Edit `.env` and set at least `NOTESNOOK_API_SECRET`. See `.env` for all configuration options.
+
+Then start the stack:
 
 ```bash
 docker compose up
 ```
 
-This takes care of setting up everything including MongoDB, Minio etc.
+This sets up MongoDB, MinIO, and the Notesnook services.
+
+Optional services (`themes-server`, `cors-proxy`, `inbox-api`): `docker compose --profile extras up`
+
+For external MongoDB, MinIO, or Garage deployments, see [`examples/`](examples/).
 
 ## TODO Self-hosting
 

Streetwriters.Common/Accessors/WampServiceAccessor.cs

@@ -0,0 +1,48 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Hosting;
+using Streetwriters.Common.Interfaces;
+
+namespace Streetwriters.Common.Accessors
+{
+    public class WampServiceAccessor(Server server) : IHostedService
+    {
+        public IUserAccountService UserAccountService { get; set; }
+        public IUserSubscriptionService? UserSubscriptionService { get; set; }
+
+        public async Task StartAsync(CancellationToken cancellationToken)
+        {
+            await InitAsync();
+        }
+
+        private async Task InitAsync()
+        {
+            this.UserAccountService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(InitAsync);
+            if (!Constants.IS_SELF_HOSTED && server != Servers.SubscriptionServer)
+            {
+                this.UserSubscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(InitAsync);
+            }
+        }
+
+        public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+    }
+}

Streetwriters.Common/Clients.cs

@@ -33,14 +33,29 @@ namespace Streetwriters.Common
         {
             Id = "notesnook",
             Name = "Notesnook",
-            SenderEmail = Constants.NOTESNOOK_SENDER_EMAIL,
+            SenderEmail = Constants.NOTESNOOK_SENDER_EMAIL ?? "noreply@notesnook.com",
             SenderName = "Notesnook",
             Type = ApplicationType.NOTESNOOK,
             AppId = ApplicationType.NOTESNOOK,
             AccountRecoveryRedirectURL = $"{Constants.NOTESNOOK_APP_HOST}/account/recovery",
             EmailConfirmedRedirectURL = $"{Constants.NOTESNOOK_APP_HOST}/account/verified",
+            PackageName = "com.streetwriters.notesnook",
             OnEmailConfirmed = async (userId) =>
             {
+                if (!Constants.IS_SELF_HOSTED)
+                {
+                    await WampServers.SubscriptionServer.PublishMessageAsync(
+                        SubscriptionServerTopics.EmailConfirmedTopic,
+                        new EmailConfirmedMessage
+                        {
+                            UserId = userId,
+                            ClientId = "notesnook",
+                            AppId = ApplicationType.NOTESNOOK,
+                            ConfirmedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
+                        }
+                    );
+                }
+
                 await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
                 {
                     UserId = userId,
@@ -58,14 +73,15 @@ namespace Streetwriters.Common
             { "notesnook", Notesnook }
         };
 
-        public static Client FindClientById(string id)
+        public static Client? FindClientById(string? id)
         {
-            if (!IsValidClient(id)) return null;
+            if (string.IsNullOrEmpty(id) || !IsValidClient(id)) return null;
             return ClientsMap[id];
         }
 
-        public static Client FindClientByAppId(ApplicationType appId)
+        public static Client? FindClientByAppId(ApplicationType? appId)
         {
+            if (appId is null) return null;
             switch (appId)
             {
                 case ApplicationType.NOTESNOOK:

Streetwriters.Common/Constants.cs

@@ -24,58 +24,75 @@ namespace Streetwriters.Common
     public class Constants
     {
         public static int COMPATIBILITY_VERSION = 1;
-        public static bool IS_SELF_HOSTED => Environment.GetEnvironmentVariable("SELF_HOSTED") == "1";
-        public static bool DISABLE_SIGNUPS => Environment.GetEnvironmentVariable("DISABLE_SIGNUPS") == "true";
-        public static string INSTANCE_NAME => Environment.GetEnvironmentVariable("INSTANCE_NAME") ?? "default";
+        public static bool IS_SELF_HOSTED => ReadSecret("SELF_HOSTED") == "1";
+        public static bool DISABLE_SIGNUPS => ReadSecret("DISABLE_SIGNUPS") == "true";
+        public static string INSTANCE_NAME => ReadSecret("INSTANCE_NAME") ?? "default";
 
         // S3 related
-        public static string S3_ACCESS_KEY => Environment.GetEnvironmentVariable("S3_ACCESS_KEY");
-        public static string S3_ACCESS_KEY_ID => Environment.GetEnvironmentVariable("S3_ACCESS_KEY_ID");
-        public static string S3_SERVICE_URL => Environment.GetEnvironmentVariable("S3_SERVICE_URL");
-        public static string S3_REGION => Environment.GetEnvironmentVariable("S3_REGION");
-        public static string S3_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_BUCKET_NAME");
-        public static string S3_INTERNAL_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_INTERNAL_BUCKET_NAME");
-        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
+        public static string S3_ACCESS_KEY => ReadSecret("S3_ACCESS_KEY") ?? throw new InvalidOperationException("S3_ACCESS_KEY is required");
+        public static string S3_ACCESS_KEY_ID => ReadSecret("S3_ACCESS_KEY_ID") ?? throw new InvalidOperationException("S3_ACCESS_KEY_ID is required");
+        public static string S3_SERVICE_URL => ReadSecret("S3_SERVICE_URL") ?? throw new InvalidOperationException("S3_SERVICE_URL is required");
+        public static string S3_REGION => ReadSecret("S3_REGION") ?? throw new InvalidOperationException("S3_REGION is required");
+        public static string S3_BUCKET_NAME => ReadSecret("S3_BUCKET_NAME") ?? throw new InvalidOperationException("S3_BUCKET_NAME is required");
+        public static string? S3_INTERNAL_BUCKET_NAME => ReadSecret("S3_INTERNAL_BUCKET_NAME");
+        public static string? S3_INTERNAL_SERVICE_URL => ReadSecret("S3_INTERNAL_SERVICE_URL");
 
         // SMTP settings
-        public static string SMTP_USERNAME => Environment.GetEnvironmentVariable("SMTP_USERNAME");
-        public static string SMTP_PASSWORD => Environment.GetEnvironmentVariable("SMTP_PASSWORD");
-        public static string SMTP_HOST => Environment.GetEnvironmentVariable("SMTP_HOST");
-        public static string SMTP_PORT => Environment.GetEnvironmentVariable("SMTP_PORT");
-        public static string SMTP_REPLYTO_EMAIL => Environment.GetEnvironmentVariable("SMTP_REPLYTO_EMAIL");
-        public static string NOTESNOOK_SENDER_EMAIL => Environment.GetEnvironmentVariable("NOTESNOOK_SENDER_EMAIL") ?? Environment.GetEnvironmentVariable("SMTP_USERNAME");
+        public static string? SMTP_USERNAME => ReadSecret("SMTP_USERNAME");
+        public static string? SMTP_PASSWORD => ReadSecret("SMTP_PASSWORD");
+        public static string? SMTP_HOST => ReadSecret("SMTP_HOST");
+        public static string? SMTP_PORT => ReadSecret("SMTP_PORT");
+        public static string? SMTP_REPLYTO_EMAIL => ReadSecret("SMTP_REPLYTO_EMAIL");
+        public static string? NOTESNOOK_SENDER_EMAIL => ReadSecret("NOTESNOOK_SENDER_EMAIL") ?? ReadSecret("SMTP_USERNAME");
 
-        public static string NOTESNOOK_APP_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_APP_HOST");
-        public static string NOTESNOOK_API_SECRET => Environment.GetEnvironmentVariable("NOTESNOOK_API_SECRET");
+        public static string? NOTESNOOK_APP_HOST => ReadSecret("NOTESNOOK_APP_HOST");
+        public static string NOTESNOOK_API_SECRET => ReadSecret("NOTESNOOK_API_SECRET") ?? throw new InvalidOperationException("NOTESNOOK_API_SECRET is required");
 
         // MessageBird is used for SMS sending
-        public static string TWILIO_ACCOUNT_SID => Environment.GetEnvironmentVariable("TWILIO_ACCOUNT_SID");
-        public static string TWILIO_AUTH_TOKEN => Environment.GetEnvironmentVariable("TWILIO_AUTH_TOKEN");
-        public static string TWILIO_SERVICE_SID => Environment.GetEnvironmentVariable("TWILIO_SERVICE_SID");
+        public static string? TWILIO_ACCOUNT_SID => ReadSecret("TWILIO_ACCOUNT_SID");
+        public static string? TWILIO_AUTH_TOKEN => ReadSecret("TWILIO_AUTH_TOKEN");
+        public static string? TWILIO_SERVICE_SID => ReadSecret("TWILIO_SERVICE_SID");
         // Server discovery
-        public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT") ?? "80");
-        public static string NOTESNOOK_SERVER_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_HOST");
-        public static string NOTESNOOK_CERT_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_PATH");
-        public static string NOTESNOOK_CERT_KEY_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_KEY_PATH");
+        public static int NOTESNOOK_SERVER_PORT => int.Parse(ReadSecret("NOTESNOOK_SERVER_PORT") ?? "80");
+        public static string? NOTESNOOK_SERVER_HOST => ReadSecret("NOTESNOOK_SERVER_HOST");
+        public static string? NOTESNOOK_CERT_PATH => ReadSecret("NOTESNOOK_CERT_PATH");
+        public static string? NOTESNOOK_CERT_KEY_PATH => ReadSecret("NOTESNOOK_CERT_KEY_PATH");
+        public static string[] KNOWN_PROXIES => (ReadSecret("KNOWN_PROXIES") ?? "").Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
 
-        public static int IDENTITY_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("IDENTITY_SERVER_PORT") ?? "80");
-        public static string IDENTITY_SERVER_HOST => Environment.GetEnvironmentVariable("IDENTITY_SERVER_HOST");
-        public static Uri IDENTITY_SERVER_URL => new(Environment.GetEnvironmentVariable("IDENTITY_SERVER_URL"));
-        public static string IDENTITY_CERT_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_PATH");
-        public static string IDENTITY_CERT_KEY_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_KEY_PATH");
+        public static int IDENTITY_SERVER_PORT => int.Parse(ReadSecret("IDENTITY_SERVER_PORT") ?? "80");
+        public static string? IDENTITY_SERVER_HOST => ReadSecret("IDENTITY_SERVER_HOST");
+        public static Uri? IDENTITY_SERVER_URL => ReadSecret("IDENTITY_SERVER_URL") is string url ? new Uri(url) : null;
+        public static string? IDENTITY_CERT_PATH => ReadSecret("IDENTITY_CERT_PATH");
+        public static string? IDENTITY_CERT_KEY_PATH => ReadSecret("IDENTITY_CERT_KEY_PATH");
 
-        public static int SSE_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SSE_SERVER_PORT") ?? "80");
-        public static string SSE_SERVER_HOST => Environment.GetEnvironmentVariable("SSE_SERVER_HOST");
-        public static string SSE_CERT_PATH => Environment.GetEnvironmentVariable("SSE_CERT_PATH");
-        public static string SSE_CERT_KEY_PATH => Environment.GetEnvironmentVariable("SSE_CERT_KEY_PATH");
+        public static int SSE_SERVER_PORT => int.Parse(ReadSecret("SSE_SERVER_PORT") ?? "80");
+        public static string? SSE_SERVER_HOST => ReadSecret("SSE_SERVER_HOST");
+        public static string? SSE_CERT_PATH => ReadSecret("SSE_CERT_PATH");
+        public static string? SSE_CERT_KEY_PATH => ReadSecret("SSE_CERT_KEY_PATH");
 
         // internal
-        public static string MONGODB_CONNECTION_STRING => Environment.GetEnvironmentVariable("MONGODB_CONNECTION_STRING");
-        public static string MONGODB_DATABASE_NAME => Environment.GetEnvironmentVariable("MONGODB_DATABASE_NAME");
-        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_PORT") ?? "80");
-        public static string SUBSCRIPTIONS_SERVER_HOST => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_HOST");
-        public static string SUBSCRIPTIONS_CERT_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_PATH");
-        public static string SUBSCRIPTIONS_CERT_KEY_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_KEY_PATH");
-        public static string[] NOTESNOOK_CORS_ORIGINS => Environment.GetEnvironmentVariable("NOTESNOOK_CORS")?.Split(",") ?? new string[] { };
+        public static string? WEBRISK_API_URI => ReadSecret("WEBRISK_API_URI");
+        public static string MONGODB_CONNECTION_STRING => ReadSecret("MONGODB_CONNECTION_STRING") ?? throw new ArgumentNullException("MONGODB_CONNECTION_STRING environment variable is not set");
+        public static string MONGODB_DATABASE_NAME => ReadSecret("MONGODB_DATABASE_NAME") ?? throw new ArgumentNullException("MONGODB_DATABASE_NAME environment variable is not set");
+        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(ReadSecret("SUBSCRIPTIONS_SERVER_PORT") ?? "80");
+        public static string? SUBSCRIPTIONS_SERVER_HOST => ReadSecret("SUBSCRIPTIONS_SERVER_HOST");
+        public static string? SUBSCRIPTIONS_CERT_PATH => ReadSecret("SUBSCRIPTIONS_CERT_PATH");
+        public static string? SUBSCRIPTIONS_CERT_KEY_PATH => ReadSecret("SUBSCRIPTIONS_CERT_KEY_PATH");
+        public static string[] NOTESNOOK_CORS_ORIGINS => ReadSecret("NOTESNOOK_CORS_ORIGINS")?.Split(",") ?? [];
+        public static string? SIGNALR_REDIS_CONNECTION_STRING => ReadSecret("SIGNALR_REDIS_CONNECTION_STRING");
+        public static string MONOGRAPH_PUBLIC_URL => ReadSecret("MONOGRAPH_PUBLIC_URL") ?? "https://monogr.ph";
+
+        public static string? ReadSecret(string name)
+        {
+            var value = Environment.GetEnvironmentVariable(name);
+            if (!string.IsNullOrEmpty(value)) return value;
+            var file = Environment.GetEnvironmentVariable(name + "_FILE");
+            if (!string.IsNullOrEmpty(file) && System.IO.File.Exists(file))
+            {
+                return System.IO.File.ReadAllText(file);
+            }
+            return null;
+        }
     }
-}
+}
+

Streetwriters.Common/Converters/InterfaceConverter.cs

@@ -1,54 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-
-namespace Streetwriters.Common.Converters
-{
-    /// <summary>
-    /// Converts simple interface into an object (assumes that there is only one class of TInterface)
-    /// </summary>
-    /// <typeparam name="TInterface">Interface type</typeparam>
-    /// <typeparam name="TClass">Class type</typeparam>
-    public class InterfaceConverter<TInterface, TClass> : JsonConverter<TInterface> where TClass : TInterface
-    {
-        public override TInterface Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
-        {
-            return JsonSerializer.Deserialize<TClass>(ref reader, options);
-        }
-
-        public override void Write(Utf8JsonWriter writer, TInterface value, JsonSerializerOptions options)
-        {
-            switch (value)
-            {
-                case null:
-                    JsonSerializer.Serialize(writer, null, options);
-                    break;
-                default:
-                    {
-                        var type = value.GetType();
-                        JsonSerializer.Serialize(writer, value, type, options);
-                        break;
-                    }
-            }
-        }
-    }
-}

Streetwriters.Common/Enums/SubscriptionPlan.cs

@@ -1,29 +1,33 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-namespace Notesnook.API.Interfaces
-{
-    public interface IEncrypted
-    {
-        string Cipher { get; set; }
-        string IV { get; set; }
-        long Length { get; set; }
-        string Salt { get; set; }
-    }
-}
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+
+namespace Streetwriters.Common.Enums
+{
+    public enum SubscriptionPlan
+    {
+        FREE = 0,
+        ESSENTIAL = 1,
+        PRO = 2,
+        BELIEVER = 3,
+        EDUCATION = 4,
+        LEGACY_PRO = 5
+    }
+}

Streetwriters.Common/Enums/SubscriptionStatus.cs

@@ -1,32 +1,30 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Collections.Generic;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Models;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IOffer : IDocument
-    {
-        ApplicationType AppId { get; set; }
-        string PromoCode { get; set; }
-        PromoCode[] Codes { get; set; }
-    }
-}
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Streetwriters.Common.Enums
+{
+    public enum SubscriptionStatus
+    {
+        ACTIVE,
+        TRIAL,
+        CANCELED,
+        PAUSED,
+        EXPIRED
+    }
+}

Streetwriters.Common/Enums/SubscriptionType.cs

@@ -1,32 +1,32 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-namespace Streetwriters.Common.Enums
-{
-    public enum SubscriptionType
-    {
-        BASIC = 0,
-        TRIAL = 1,
-        BETA = 2,
-        PREMIUM = 5,
-        PREMIUM_EXPIRED = 6,
-        PREMIUM_CANCELED = 7,
-        PREMIUM_PAUSED = 8
-    }
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Streetwriters.Common.Enums
+{
+    public enum SubscriptionType
+    {
+        BASIC = 0,
+        TRIAL = 1,
+        BETA = 2,
+        PREMIUM = 5,
+        PREMIUM_EXPIRED = 6,
+        PREMIUM_CANCELED = 7,
+        PREMIUM_PAUSED = 8,
+    }
 }

Streetwriters.Common/Extensions/AppBuilderExtensions.cs

@@ -19,10 +19,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Collections.Generic;
+using System.Net;
 using System.Text.Json;
 using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
 using WampSharp.AspNetCore.WebSockets.Server;
 using WampSharp.Binding;
 using WampSharp.V2;
@@ -42,7 +46,7 @@ namespace Streetwriters.Common.Extensions
                     var data = new Dictionary<string, object>
                     {
                         { "version", Constants.COMPATIBILITY_VERSION },
-                        { "id", server.Id },
+                        { "id", server.Id ?? "unknown" },
                         { "instance", Constants.INSTANCE_NAME }
                     };
                     await context.Response.WriteAsync(JsonSerializer.Serialize(data));
@@ -51,9 +55,9 @@ namespace Streetwriters.Common.Extensions
             return app;
         }
 
-        public static IApplicationBuilder UseWamp<T>(this IApplicationBuilder app, WampServer<T> server, Action<IWampHostedRealm, WampServer<T>> action) where T : new()
+        public static IApplicationBuilder UseWamp(this IApplicationBuilder app, WampServer server, Action<IWampHostedRealm, WampServer> action)
         {
-            WampHost host = new WampHost();
+            WampHost host = new();
 
             app.Map(server.Endpoint, builder =>
             {
@@ -70,17 +74,40 @@ namespace Streetwriters.Common.Extensions
             return app;
         }
 
-        public static T GetService<T>(this IApplicationBuilder app)
+        public static T GetService<T>(this IApplicationBuilder app) where T : notnull
         {
             return app.ApplicationServices.GetRequiredService<T>();
         }
 
-        public static T GetScopedService<T>(this IApplicationBuilder app)
+        public static T GetScopedService<T>(this IApplicationBuilder app) where T : notnull
         {
-            using (var scope = app.ApplicationServices.CreateScope())
+            using var scope = app.ApplicationServices.CreateScope();
+            return scope.ServiceProvider.GetRequiredService<T>();
+        }
+
+        public static IApplicationBuilder UseForwardedHeadersWithKnownProxies(this IApplicationBuilder app, IWebHostEnvironment env, string forwardedForHeaderName = null)
+        {
+            if (!env.IsDevelopment())
             {
-                return scope.ServiceProvider.GetRequiredService<T>();
+                var forwardedHeadersOptions = new ForwardedHeadersOptions
+                {
+                    ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto
+                };
+
+                if (!string.IsNullOrEmpty(forwardedForHeaderName))
+                {
+                    forwardedHeadersOptions.ForwardedForHeaderName = forwardedForHeaderName;
+                }
+
+                foreach (var proxy in Constants.KNOWN_PROXIES)
+                {
+                    forwardedHeadersOptions.KnownProxies.Add(IPAddress.Parse(proxy));
+                }
+
+                app.UseForwardedHeaders(forwardedHeadersOptions);
             }
+
+            return app;
         }
     }
 }

Streetwriters.Common/Extensions/HttpClientExtensions.cs

@@ -17,6 +17,7 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Json;
@@ -28,7 +29,7 @@ namespace Streetwriters.Common.Extensions
 {
     public static class HttpClientExtensions
     {
-        public static async Task<T> SendRequestAsync<T>(this HttpClient httpClient, string url, IHeaderDictionary headers, HttpMethod method, HttpContent content = null) where T : IResponse, new()
+        public static async Task<T> SendRequestAsync<T>(this HttpClient httpClient, string url, IHeaderDictionary? headers, HttpMethod method, HttpContent? content = null) where T : IResponse, new()
         {
             var request = new HttpRequestMessage(method, url);
 
@@ -37,34 +38,37 @@ namespace Streetwriters.Common.Extensions
                 request.Content = content;
             }
 
-            foreach (var header in headers)
+            if (headers != null)
             {
-                if (header.Key == "Content-Type" || header.Key == "Content-Length")
+                foreach (var header in headers)
                 {
-                    if (request.Content != null)
-                        request.Content.Headers.TryAddWithoutValidation(header.Key, header.Value.AsEnumerable());
-                    continue;
+                    if (header.Key == "Content-Type" || header.Key == "Content-Length")
+                    {
+                        request.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value.AsEnumerable());
+                        continue;
+                    }
+                    request.Headers.TryAddWithoutValidation(header.Key, header.Value.AsEnumerable());
                 }
-                request.Headers.TryAddWithoutValidation(header.Key, header.Value.AsEnumerable());
             }
 
             var response = await httpClient.SendAsync(request);
-            if (response.Content.Headers.ContentLength > 0)
+            if (response.Content.Headers.ContentLength > 0 && response.Content.Headers.ContentType?.ToString()?.Contains("application/json") == true)
             {
                 var res = await response.Content.ReadFromJsonAsync<T>();
-                res.Success = response.IsSuccessStatusCode;
-                res.StatusCode = (int)response.StatusCode;
-                return res;
-            }
-            else
-            {
-                return new T { Success = response.IsSuccessStatusCode, StatusCode = (int)response.StatusCode };
+                if (res != null)
+                {
+                    res.Success = response.IsSuccessStatusCode;
+                    res.StatusCode = (int)response.StatusCode;
+                    return res;
+                }
             }
+
+            return new T { Success = response.IsSuccessStatusCode, StatusCode = (int)response.StatusCode, Content = response.Content };
         }
 
         public static Task<T> ForwardAsync<T>(this HttpClient httpClient, IHttpContextAccessor accessor, string url, HttpMethod method) where T : IResponse, new()
         {
-            var httpContext = accessor.HttpContext;
+            var httpContext = accessor.HttpContext ?? throw new InvalidOperationException("HttpContext is not available");
             var content = new StreamContent(httpContext.Request.BodyReader.AsStream());
             return httpClient.SendRequestAsync<T>(url, httpContext.Request.Headers, method, content);
         }

Streetwriters.Common/Extensions/ServiceCollectionServiceExtensions.cs

@@ -18,6 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using Microsoft.Extensions.DependencyInjection;
+using Streetwriters.Common.Accessors;
 using Streetwriters.Data.DbContexts;
 using Streetwriters.Data.Repositories;
 
@@ -25,9 +26,16 @@ namespace Streetwriters.Common.Extensions
 {
     public static class ServiceCollectionServiceExtensions
     {
+        public static IServiceCollection AddWampServiceAccessor(this IServiceCollection services, Server server)
+        {
+            services.AddSingleton<WampServiceAccessor>((provider) => new WampServiceAccessor(server));
+            services.AddHostedService(provider => provider.GetRequiredService<WampServiceAccessor>());
+            return services;
+        }
+
         public static IServiceCollection AddRepository<T>(this IServiceCollection services, string collectionName, string database) where T : class
         {
-            services.AddSingleton((provider) => MongoDbContext.GetMongoCollection<T>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
+            services.AddSingleton((provider) => MongoDbContext.GetMongoCollection<T>(provider.GetRequiredService<MongoDB.Driver.IMongoClient>(), database, collectionName));
             services.AddScoped<Repository<T>>();
             return services;
         }
@@ -44,7 +52,8 @@ namespace Streetwriters.Common.Extensions
                         b.WithOrigins(Constants.NOTESNOOK_CORS_ORIGINS);
 
                     b.AllowAnyMethod()
-                    .AllowAnyHeader();
+                    .AllowAnyHeader()
+                    .WithExposedHeaders(["X-Object-Size", "Content-Length"]);
                 });
             });
             return services;

Streetwriters.Common/Extensions/WampRealmExtensions.cs

@@ -18,6 +18,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
+using System.Reactive.Disposables;
+using System.Threading;
 using Microsoft.AspNetCore.Builder;
 using Streetwriters.Common.Interfaces;
 using WampSharp.AspNetCore.WebSockets.Server;
@@ -38,5 +40,27 @@ namespace Streetwriters.Common.Extensions
         {
             return realm.Services.GetSubject<T>(topicName).Subscribe<T>(async (message) => await handler.Process(message));
         }
+
+        public static IDisposable SubscribeWithSemaphore<T>(this IWampHostedRealm realm, string topicName, IMessageHandler<T> handler)
+        {
+            var semaphore = new SemaphoreSlim(1, 1);
+            var subscriber = realm.Services.GetSubject<T>(topicName).Subscribe<T>(async (message) =>
+            {
+                await semaphore.WaitAsync();
+                try
+                {
+                    await handler.Process(message);
+                }
+                finally
+                {
+                    semaphore.Release();
+                }
+            });
+            return Disposable.Create(() =>
+            {
+                subscriber.Dispose();
+                semaphore.Dispose();
+            });
+        }
     }
 }

Streetwriters.Common/Helpers/FeatureAuthorizationHelper.cs

@@ -0,0 +1,62 @@
+using System.IO;
+using System.Security.Claims;
+using System.Threading.Tasks;
+using Streetwriters.Common.Enums;
+using Streetwriters.Common.Interfaces;
+using WebMarkupMin.Core;
+using WebMarkupMin.Core.Loggers;
+
+namespace Streetwriters.Common.Helpers
+{
+    public enum Features
+    {
+        SMS_2FA,
+        MONOGRAPH_ANALYTICS
+    }
+
+    public static class FeatureAuthorizationHelper
+    {
+        private static SubscriptionPlan? GetUserSubscriptionPlan(string clientId, ClaimsPrincipal user)
+        {
+            var claimKey = $"{clientId}:status";
+            var status = user.FindFirstValue(claimKey);
+            switch (status)
+            {
+                case "free":
+                    return SubscriptionPlan.FREE;
+                case "believer":
+                    return SubscriptionPlan.BELIEVER;
+                case "education":
+                    return SubscriptionPlan.EDUCATION;
+                case "essential":
+                    return SubscriptionPlan.ESSENTIAL;
+                case "pro":
+                    return SubscriptionPlan.PRO;
+                case "legacy_pro":
+                    return SubscriptionPlan.LEGACY_PRO;
+                default:
+                    return null;
+            }
+        }
+
+        public static bool IsFeatureAllowed(Features feature, string clientId, ClaimsPrincipal user)
+        {
+            if (Constants.IS_SELF_HOSTED)
+                return true;
+
+            var status = GetUserSubscriptionPlan(clientId, user);
+
+            switch (feature)
+            {
+                case Features.SMS_2FA:
+                case Features.MONOGRAPH_ANALYTICS:
+                    return status == SubscriptionPlan.LEGACY_PRO ||
+                            status == SubscriptionPlan.PRO ||
+                            status == SubscriptionPlan.EDUCATION ||
+                            status == SubscriptionPlan.BELIEVER;
+                default:
+                    return false;
+            }
+        }
+    }
+}

Streetwriters.Common/Helpers/WampHelper.cs

@@ -1,47 +1,68 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Reactive.Subjects;
-using System.Threading.Tasks;
-using Streetwriters.Common.Messages;
-using WampSharp.V2;
-using WampSharp.V2.Client;
-
-namespace Streetwriters.Common.Helpers
-{
-    public class WampHelper
-    {
-        public static async Task<IWampRealmProxy> OpenWampChannelAsync(string server, string realmName)
-        {
-            DefaultWampChannelFactory channelFactory = new();
-
-            IWampChannel channel = channelFactory.CreateJsonChannel(server, realmName);
-
-            await channel.Open();
-
-            return channel.RealmProxy;
-        }
-
-        public static void PublishMessage<T>(IWampRealmProxy realm, string topicName, T message)
-        {
-            var subject = realm.Services.GetSubject<T>(topicName);
-            subject.OnNext(message);
-        }
-    }
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Collections.Generic;
+using System.Reactive.Subjects;
+using System.Threading.Tasks;
+using Streetwriters.Common.Messages;
+using WampSharp.V2;
+using WampSharp.V2.Client;
+
+namespace Streetwriters.Common.Helpers
+{
+    public class WampHelper
+    {
+        public static async Task<IWampChannel> OpenWampChannelAsync(string server, string realmName)
+        {
+            DefaultWampChannelFactory channelFactory = new();
+
+            IWampChannel channel = channelFactory.CreateJsonChannel(server, realmName);
+
+            var isConnected = false;
+            while (!isConnected)
+            {
+                try
+                {
+                    await channel.Open();
+                    isConnected = true;
+                }
+                catch
+                {
+                    await Task.Delay(5000);
+                    continue;
+                }
+            }
+
+            return channel;
+        }
+
+        public static void PublishMessage<T>(IWampRealmProxy realm, string topicName, T message)
+        {
+            var subject = realm.Services.GetSubject<T>(topicName);
+            subject.OnNext(message);
+        }
+
+        public static void PublishMessages<T>(IWampRealmProxy realm, string topicName, IEnumerable<T> messages)
+        {
+            var subject = realm.Services.GetSubject<T>(topicName);
+            foreach (var message in messages)
+                subject.OnNext(message);
+        }
+    }
 }

Streetwriters.Common/Interfaces/IClient.cs

@@ -33,6 +33,6 @@ namespace Streetwriters.Common.Interfaces
         string SenderName { get; set; }
         string EmailConfirmedRedirectURL { get; }
         string AccountRecoveryRedirectURL { get; }
-        Func<string, Task> OnEmailConfirmed { get; set; }
+        Func<string, Task>? OnEmailConfirmed { get; set; }
     }
 }

Streetwriters.Common/Interfaces/IDocument.cs

@@ -17,11 +17,13 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using MongoDB.Bson;
+
 namespace Streetwriters.Common.Interfaces
 {
     public interface IDocument
     {
-        string Id
+        ObjectId Id
         {
             get; set;
         }

Streetwriters.Common/Interfaces/IEmailSender.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Net.Mail;
 using System.Threading.Tasks;
 using MimeKit;
 using MimeKit.Cryptography;
@@ -11,9 +12,9 @@ namespace Streetwriters.Common.Interfaces
         Task SendEmailAsync(
             string email,
             EmailTemplate template,
-            IClient client,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
+            MailAddress from,
+            GnuPGContext? gpgContext = null,
+            Dictionary<string, byte[]>? attachments = null
         );
     }
 }

Streetwriters.Common/Interfaces/IResponse.cs

@@ -17,11 +17,14 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.Net.Http;
+
 namespace Streetwriters.Common.Interfaces
 {
     public interface IResponse
     {
         bool Success { get; set; }
         int StatusCode { get; set; }
+        HttpContent? Content { get; set; }
     }
 }

Streetwriters.Common/Interfaces/ISubscription.cs

@@ -1,41 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.ComponentModel.DataAnnotations;
-using System.Runtime.Serialization;
-using Streetwriters.Common.Attributes;
-using Streetwriters.Common.Converters;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Models;
-
-namespace Streetwriters.Common.Interfaces
-{
-    [JsonInterfaceConverter(typeof(InterfaceConverter<ISubscription, Subscription>))]
-    public interface ISubscription : IDocument
-    {
-        string UserId { get; set; }
-        ApplicationType AppId { get; set; }
-        SubscriptionProvider Provider { get; set; }
-        long StartDate { get; set; }
-        long ExpiryDate { get; set; }
-        SubscriptionType Type { get; set; }
-        string OrderId { get; set; }
-        string SubscriptionId { get; set; }
-    }
-}

Streetwriters.Common/Interfaces/IURLAnalyzer.cs

@@ -0,0 +1,13 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using MimeKit;
+using MimeKit.Cryptography;
+using Streetwriters.Common.Models;
+
+namespace Streetwriters.Common.Interfaces
+{
+    public interface IURLAnalyzer
+    {
+        Task<bool> IsURLSafeAsync(string uri);
+    }
+}

Streetwriters.Common/Interfaces/IUserAccountService.cs

@@ -7,10 +7,16 @@ namespace Streetwriters.Common.Interfaces
     public interface IUserAccountService
     {
         [WampProcedure("co.streetwriters.identity.users.get_user")]
-        Task<UserModel> GetUserAsync(string clientId, string userId);
+        Task<UserModel?> GetUserAsync(string clientId, string userId);
         [WampProcedure("co.streetwriters.identity.users.delete_user")]
         Task DeleteUserAsync(string clientId, string userId, string password);
-        // [WampProcedure("co.streetwriters.identity.users.create_user")]
-        // Task<UserModel> CreateUserAsync();
+        [WampProcedure("co.streetwriters.identity.users.change_password")]
+        Task<bool> ChangePasswordAsync(string userId, string oldPassword, string newPassword);
+        [WampProcedure("co.streetwriters.identity.users.reset_password")]
+        Task<bool> ResetPasswordAsync(string userId, string newPassword);
+        [WampProcedure("co.streetwriters.identity.users.clear_sessions")]
+        Task<bool> ClearSessionsAsync(string userId, string clientId, bool all, string jti, string? refreshToken);
+        [WampProcedure("co.streetwriters.identity.users.create_user")]
+        Task<SignupResponse> CreateUserAsync(string clientId, string email, string password, string? userAgent = null);
     }
 }

Streetwriters.Common/Interfaces/IUserSubscriptionService.cs

@@ -8,6 +8,9 @@ namespace Streetwriters.Common.Interfaces
     public interface IUserSubscriptionService
     {
         [WampProcedure("co.streetwriters.subscriptions.subscriptions.get_user_subscription")]
-        Task<Subscription> GetUserSubscriptionAsync(string clientId, string userId);
+        Task<Subscription?> GetUserSubscriptionAsync(string clientId, string userId);
+        [WampProcedure("co.streetwriters.subscriptions.subscriptions.is_user_subscribed")]
+        Task<bool> IsUserSubscribedAsync(string clientId, string userId);
+        Subscription TransformUserSubscription(Subscription subscription);
     }
 }

Streetwriters.Common/Logger.cs

@@ -1,52 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.IO;
-using System.Threading.Tasks;
-
-namespace Streetwriters.Common
-{
-    public class Slogger<T>
-    {
-        public static Task Info(string scope, params string[] messages)
-        {
-            return Write(Format("info", scope, messages));
-        }
-
-        public static Task Error(string scope, params string[] messages)
-        {
-            return Write(Format("error", scope, messages));
-        }
-        private static string Format(string level, string scope, params string[] messages)
-        {
-            var date = DateTime.UtcNow.ToString("MM-dd-yyyy HH:mm:ss");
-            var messageText = string.Join(" ", messages);
-            return $"[{date}] | {level} | <{scope}> {messageText}";
-        }
-        private static Task Write(string line)
-        {
-            var logDirectory = Path.GetFullPath("./logs");
-            if (!Directory.Exists(logDirectory))
-                Directory.CreateDirectory(logDirectory);
-            var path = Path.Join(logDirectory, typeof(T).FullName + "-" + DateTime.UtcNow.ToString("MM-dd-yyyy") + ".log");
-            return File.AppendAllLinesAsync(path, new string[1] { line });
-        }
-    }
-}

Streetwriters.Common/Messages/CreateSubscriptionMessage.cs

@@ -1,63 +1,66 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System.Runtime.Serialization;
-using System.Text.Json.Serialization;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
-
-namespace Streetwriters.Common.Messages
-{
-    public class CreateSubscriptionMessage
-    {
-        [JsonPropertyName("userId")]
-        public string UserId { get; set; }
-
-        [JsonPropertyName("provider")]
-        public SubscriptionProvider Provider { get; set; }
-
-        [JsonPropertyName("appId")]
-        public ApplicationType AppId { get; set; }
-
-        [JsonPropertyName("type")]
-        public SubscriptionType Type { get; set; }
-
-        [JsonPropertyName("start")]
-        public long StartTime { get; set; }
-
-        [JsonPropertyName("expiry")]
-        public long ExpiryTime { get; set; }
-
-        [JsonPropertyName("orderId")]
-        public string OrderId { get; set; }
-
-        [JsonPropertyName("updateURL")]
-        public string UpdateURL { get; set; }
-
-        [JsonPropertyName("cancelURL")]
-        public string CancelURL { get; set; }
-
-        [JsonPropertyName("subscriptionId")]
-        public string SubscriptionId { get; set; }
-
-        [JsonPropertyName("productId")]
-        public string ProductId { get; set; }
-        public bool Extend { get; set; }
-    }
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using Streetwriters.Common.Enums;
+using Streetwriters.Common.Interfaces;
+
+namespace Streetwriters.Common.Messages
+{
+    public class CreateSubscriptionMessage
+    {
+        [JsonPropertyName("userId")]
+        public required string UserId { get; set; }
+
+        [JsonPropertyName("provider")]
+        public SubscriptionProvider Provider { get; set; }
+
+        [JsonPropertyName("appId")]
+        public ApplicationType AppId { get; set; }
+
+        [JsonPropertyName("type")]
+        public SubscriptionType Type { get; set; }
+
+        [JsonPropertyName("start")]
+        public long StartTime { get; set; }
+
+        [JsonPropertyName("expiry")]
+        public long ExpiryTime { get; set; }
+
+        [JsonPropertyName("orderId")]
+        public string? OrderId { get; set; }
+
+        [JsonPropertyName("updateURL")]
+        public string? UpdateURL { get; set; }
+
+        [JsonPropertyName("cancelURL")]
+        public string? CancelURL { get; set; }
+
+        [JsonPropertyName("subscriptionId")]
+        public string? SubscriptionId { get; set; }
+
+        [JsonPropertyName("productId")]
+        public string? ProductId { get; set; }
+
+        [JsonPropertyName("extend")]
+        public bool Extend { get; set; }
+    }
 }

Streetwriters.Common/Messages/CreateSubscriptionMessageV2.cs

@@ -0,0 +1,69 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+using System.Runtime.Serialization;
+using System.Text.Json.Serialization;
+using Streetwriters.Common.Enums;
+using Streetwriters.Common.Interfaces;
+
+namespace Streetwriters.Common.Messages
+{
+    public class CreateSubscriptionMessageV2
+    {
+        [JsonPropertyName("userId")]
+        public required string UserId { get; set; }
+
+        [JsonPropertyName("provider")]
+        public SubscriptionProvider Provider { get; set; }
+
+        [JsonPropertyName("appId")]
+        public ApplicationType AppId { get; set; }
+
+        [JsonPropertyName("plan")]
+        public SubscriptionPlan Plan { get; set; }
+
+        [JsonPropertyName("status")]
+        public SubscriptionStatus Status { get; set; }
+
+        [JsonPropertyName("start")]
+        public long StartTime { get; set; }
+
+        [JsonPropertyName("expiry")]
+        public long ExpiryTime { get; set; }
+
+        [JsonPropertyName("orderId")]
+        public string? OrderId { get; set; }
+
+        [JsonPropertyName("subscriptionId")]
+        public string? SubscriptionId { get; set; }
+
+        [JsonPropertyName("productId")]
+        public string? ProductId { get; set; }
+
+        [JsonPropertyName("timestamp")]
+        public long Timestamp { get; set; }
+
+        [JsonPropertyName("trialExpiry")]
+        public long TrialExpiryTime { get; set; }
+
+        [JsonPropertyName("googlePurchaseToken")]
+        public string? GooglePurchaseToken { get; set; }
+    }
+}

Streetwriters.Common/Messages/DeleteSubscriptionMessage.cs

@@ -27,7 +27,7 @@ namespace Streetwriters.Common.Messages
     public class DeleteSubscriptionMessage
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("appId")]
         public ApplicationType AppId { get; set; }

Streetwriters.Common/Messages/DeleteUserMessage.cs

@@ -27,6 +27,6 @@ namespace Streetwriters.Common.Messages
     public class DeleteUserMessage
     {
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
     }
 }

Streetwriters.Common/Messages/EmailConfirmedMessage.cs

@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+using Streetwriters.Common.Enums;
+
+namespace Streetwriters.Common.Messages
+{
+    public class EmailConfirmedMessage
+    {
+        [JsonPropertyName("userId")]
+        public required string UserId { get; set; }
+
+        [JsonPropertyName("clientId")]
+        public required string ClientId { get; set; }
+
+        [JsonPropertyName("appId")]
+        public ApplicationType AppId { get; set; }
+
+        [JsonPropertyName("confirmedAt")]
+        public long ConfirmedAt { get; set; }
+    }
+}

Streetwriters.Common/Messages/SendSSEMessage.cs

@@ -26,10 +26,10 @@ namespace Streetwriters.Common.Messages
     public class Message
     {
         [JsonPropertyName("type")]
-        public string Type { get; set; }
+        public required string Type { get; set; }
 
         [JsonPropertyName("data")]
-        public string Data { get; set; }
+        public string? Data { get; set; }
     }
     public class SendSSEMessage
     {
@@ -37,12 +37,12 @@ namespace Streetwriters.Common.Messages
         public bool SendToAll { get; set; }
 
         [JsonPropertyName("userId")]
-        public string UserId { get; set; }
+        public required string UserId { get; set; }
 
         [JsonPropertyName("message")]
-        public Message Message { get; set; }
+        public required Message Message { get; set; }
 
         [JsonPropertyName("originTokenId")]
-        public string OriginTokenId { get; set; }
+        public string? OriginTokenId { get; set; }
     }
 }

Streetwriters.Common/Models/Client.cs

@@ -31,15 +31,16 @@ namespace Streetwriters.Common.Models
 {
     public class Client : IClient
     {
-        public string Id { get; set; }
-        public string Name { get; set; }
+        public required string Id { get; set; }
+        public required string Name { get; set; }
         public ApplicationType Type { get; set; }
         public ApplicationType AppId { get; set; }
-        public string SenderEmail { get; set; }
-        public string SenderName { get; set; }
-        public string EmailConfirmedRedirectURL { get; set; }
-        public string AccountRecoveryRedirectURL { get; set; }
+        public required string SenderEmail { get; set; }
+        public required string SenderName { get; set; }
+        public required string EmailConfirmedRedirectURL { get; set; }
+        public required string AccountRecoveryRedirectURL { get; set; }
+        public required string PackageName { get; set; }
 
-        public Func<string, Task> OnEmailConfirmed { get; set; }
+        public Func<string, Task>? OnEmailConfirmed { get; set; }
     }
 }

Streetwriters.Common/Models/EmailTemplate.cs

@@ -3,9 +3,9 @@ namespace Streetwriters.Common.Models
     public class EmailTemplate
     {
         public int? Id { get; set; }
-        public object Data { get; set; }
-        public string Subject { get; set; }
-        public string Html { get; set; }
-        public string Text { get; set; }
+        public object? Data { get; set; }
+        public required string Subject { get; set; }
+        public required string Html { get; set; }
+        public required string Text { get; set; }
     }
 }

Streetwriters.Common/Models/GiftCard.cs

@@ -9,14 +9,14 @@ namespace Streetwriters.Common.Models
     {
         public GiftCard()
         {
-            Id = ObjectId.GenerateNewId().ToString();
+            Id = ObjectId.GenerateNewId();
         }
 
-        public string Code { get; set; }
-        public string OrderId { get; set; }
-        public string OrderIdType { get; set; }
-        public string ProductId { get; set; }
-        public string RedeemedBy { get; set; }
+        public required string Code { get; set; }
+        public required string OrderId { get; set; }
+        public required string OrderIdType { get; set; }
+        public required string ProductId { get; set; }
+        public string? RedeemedBy { get; set; }
         public long RedeemedAt { get; set; }
         public long Timestamp { get; set; }
         public long Term { get; set; }
@@ -24,6 +24,6 @@ namespace Streetwriters.Common.Models
         [BsonId]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
-        public string Id { get; set; }
+        public ObjectId Id { get; set; }
     }
 }