sync: replace mongodb with file system based repository

.env

@@ -1,72 +1,31 @@
-# Description: Name of your self hosted instance. Used in the client apps for identification purposes
-# Required: yes
-# Example: notesnook-instance-sg
-INSTANCE_NAME=self-hosted-notesnook-instance
+# Required variables
+NOTESNOOK_API_SECRET= # This should be a randomly generated secret
 
-# Description: This secret is used for generating, validating, and introspecting auth tokens. It must be a randomly generated token (preferably >32 characters).
-# Required: yes
-NOTESNOOK_API_SECRET=
-
-# Description: Use this flag to disable creation of new accounts on your instance (i.e. in case it is exposed to the Internet).
-# Required: yes
-# Possible values: true/false
-DISABLE_SIGNUPS=false
-
-### SMTP Configuration ###
-# SMTP Configuration is required for sending emails for password reset, 2FA emails etc. You can get SMTP settings from your email provider.
-
-# Description: Username for the SMTP connection (most time it is the email address of your account). Check your email provider's documentation to get the appropriate value.
-# Required: yes
+# SMTP settings required for delivering emails
 SMTP_USERNAME=
-# Description: Password for the SMTP connection. Check your email provider's documentation to get the appropriate value.
-# Required: yes
 SMTP_PASSWORD=
-# Description: Host on which the the SMTP connection is running. Check your email provider's documentation to get the appropriate value.
-# Required: yes
-# Example: smtp.gmail.com
 SMTP_HOST=
-# Description: Port on which the the SMTP connection is running. Check your email provider's documentation to get the appropriate value.
-# Required: yes
-# Example: 465
 SMTP_PORT=
+NOTESNOOK_SENDER_EMAIL=
+NOTESNOOK_SENDER_NAME=
+SMTP_REPLYTO_NAME= # optional
+SMTP_REPLYTO_EMAIL= # optional
 
-# Description: Twilio account SID is required for sending SMS with 2FA codes. Learn more here: https://help.twilio.com/articles/14726256820123-What-is-a-Twilio-Account-SID-and-where-can-I-find-it-
-# Required: no
-TWILIO_ACCOUNT_SID=
-# Description: Twilio account auth is required for sending SMS with 2FA codes. Learn more here: https://help.twilio.com/articles/223136027-Auth-Tokens-and-How-to-Change-Them
-# Required: no
-TWILIO_AUTH_TOKEN=
-# Description: The unique string that we created to identify the Service resource.
-# Required: no
-# Example: VAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-TWILIO_SERVICE_SID=
+# MessageBird is used for 2FA via SMS
+MESSAGEBIRD_ACCESS_KEY=
 
-# Description: Add the origins for which you want to allow CORS. Leave it empty to allow all origins to access your server. If you want to allow multiple origins, seperate each origin with a comma.
-# Required: no
-# Example: https://app.notesnook.com,http://localhost:3000
-NOTESNOOK_CORS_ORIGINS=
+# Server discovery settings
+# The domain must be without protocol
+# e.g. example.org NOT http://example.org
+NOTESNOOK_SERVER_DOMAIN=
+IDENTITY_SERVER_DOMAIN=
+SSE_SERVER_DOMAIN=
 
-# Description: This is the public URL for the web app, and is used by the backend for creating redirect URLs (e.g. after email confirmation etc). 
-# Note: the URL has no slashes at the end
-# Required: yes
-# Example: https://app.notesnook.com
-NOTESNOOK_APP_PUBLIC_URL=https://app.notesnook.com
-# Description: This is the public URL for the monograph frontend.
-# Required: yes
-# Example: https://monogr.ph
-MONOGRAPH_PUBLIC_URL=http://localhost:6264
-# Description: This is the public URL for the Authentication server. Used for generating email confirmation & password reset URLs.
-# Required: yes
-# Example: https://auth.streetwriters.co
-AUTH_SERVER_PUBLIC_URL=http://localhost:8264
-# Description: This is the public URL for the S3 attachments server (minio). It'll be used by the Notesnook clients for uploading/downloading attachments.
-# Required: yes
-# Example: https://attachments.notesnook.com
-ATTACHMENTS_SERVER_PUBLIC_URL=http://localhost:9000
+# url of the web app instance you want to use
+# e.g. http://localhost:3000
+# Note: no slashes at the end
+NOTESNOOK_APP_HOST=
 
-# Description: Custom username for the root Minio account. Minio is used for storing your attachments. This must be greater than 3 characters in length.
-# Required: no
-MINIO_ROOT_USER=
-# Description: Custom password for the root Minio account. Minio is used for storing your attachments. This must be greater than 8 characters in length.
-# Required: no
-MINIO_ROOT_PASSWORD=
+# Minio is used for S3 storage
+MINIO_ROOT_USER= # aka. AccessKeyId (must be > 3 characters)
+MINIO_ROOT_PASSWORD= # aka. AccessKey (must be > 8 characters)

.github/workflows/publish.yml

@@ -1,85 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-# GitHub recommends pinning actions to a commit SHA.
-# To get a newer version, you will need to update the SHA.
-# You can also reference a tag or branch, but the action may change without warning.
-
-name: Publish Docker images
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  push_to_registry:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        repos:
-          - image: streetwriters/notesnook-sync
-            file: ./Notesnook.API/Dockerfile
-
-          - image: streetwriters/identity
-            file: ./Streetwriters.Identity/Dockerfile
-
-          - image: streetwriters/sse
-            file: ./Streetwriters.Messenger/Dockerfile
-    permissions:
-      packages: write
-      contents: read
-      attestations: write
-      id-token: write
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-
-      # Setup Buildx
-      - name: Docker Setup Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          ecr: auto
-          logout: true
-
-      # Pull previous image from docker hub to use it as cache to improve the image build time.
-      - name: docker pull cache image
-        continue-on-error: true
-        run: docker pull ${{ matrix.repos.image }}:latest
-
-      # Setup QEMU
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ matrix.repos.image }}
-
-      - name: Build and push Docker image
-        id: push
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ${{ matrix.repos.file }}
-          push: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
-          tags: ${{ steps.meta.outputs.tags }}
-          cache-from: ${{ matrix.repos.image }}:latest
-
-      - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@v1
-        with:
-          subject-name: index.docker.io/${{ matrix.repos.image }}
-          subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true

.gitignore

@@ -262,6 +262,6 @@ __pycache__/
 
 keys/
 dist/
+appsettings.json
 keystore/
 .env.local
-Notesnook.API/sync/

.vscode/launch.json

@@ -9,7 +9,8 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-notesnook",
-      "program": "bin/Debug/net8.0/Notesnook.API.dll",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/Notesnook.API/bin/Debug/net7.0/linux-x64/Notesnook.API.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Notesnook.API",
       "stopAtEntry": false,
@@ -24,7 +25,8 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-identity",
-      "program": "bin/Debug/net8.0/Streetwriters.Identity.dll",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/Streetwriters.Identity/bin/Debug/net7.0/linux-x64/Streetwriters.Identity.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Identity",
       "stopAtEntry": false,
@@ -39,7 +41,8 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-messenger",
-      "program": "bin/Debug/net8.0/Streetwriters.Messenger.dll",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/Streetwriters.Messenger/bin/Debug/net7.0/linux-x64/Streetwriters.Messenger.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Messenger",
       "stopAtEntry": false,

Notesnook.API/Accessors/SyncItemsRepositoryAccessor.cs

@@ -17,76 +17,47 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using Microsoft.Extensions.DependencyInjection;
-using MongoDB.Driver;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Repositories;
-using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Accessors
 {
     public class SyncItemsRepositoryAccessor : ISyncItemsRepositoryAccessor
     {
-        public SyncItemsRepository Notes { get; }
-        public SyncItemsRepository Notebooks { get; }
-        public SyncItemsRepository Shortcuts { get; }
-        public SyncItemsRepository Relations { get; }
-        public SyncItemsRepository Reminders { get; }
-        public SyncItemsRepository Contents { get; }
-        public SyncItemsRepository LegacySettings { get; }
-        public SyncItemsRepository Settings { get; }
-        public SyncItemsRepository Attachments { get; }
-        public SyncItemsRepository Colors { get; }
-        public SyncItemsRepository Vaults { get; }
-        public SyncItemsRepository Tags { get; }
+        public SyncItemsRepository<Note> Notes { get; }
+        public SyncItemsRepository<Notebook> Notebooks { get; }
+        public SyncItemsRepository<Shortcut> Shortcuts { get; }
+        public SyncItemsRepository<Relation> Relations { get; }
+        public SyncItemsRepository<Reminder> Reminders { get; }
+        public SyncItemsRepository<Content> Contents { get; }
+        public SyncItemsRepository<Setting> Settings { get; }
+        public SyncItemsRepository<Attachment> Attachments { get; }
         public Repository<UserSettings> UsersSettings { get; }
         public Repository<Monograph> Monographs { get; }
 
-        public SyncItemsRepositoryAccessor(IDbContext dbContext,
-
-        [FromKeyedServices(Collections.NotebooksKey)]
-        IMongoCollection<SyncItem> notebooks,
-        [FromKeyedServices(Collections.NotesKey)]
-        IMongoCollection<SyncItem> notes,
-        [FromKeyedServices(Collections.ContentKey)]
-        IMongoCollection<SyncItem> content,
-        [FromKeyedServices(Collections.SettingsKey)]
-        IMongoCollection<SyncItem> settings,
-        [FromKeyedServices(Collections.LegacySettingsKey)]
-        IMongoCollection<SyncItem> legacySettings,
-        [FromKeyedServices(Collections.AttachmentsKey)]
-        IMongoCollection<SyncItem> attachments,
-        [FromKeyedServices(Collections.ShortcutsKey)]
-        IMongoCollection<SyncItem> shortcuts,
-        [FromKeyedServices(Collections.RemindersKey)]
-        IMongoCollection<SyncItem> reminders,
-        [FromKeyedServices(Collections.RelationsKey)]
-        IMongoCollection<SyncItem> relations,
-        [FromKeyedServices(Collections.ColorsKey)]
-        IMongoCollection<SyncItem> colors,
-        [FromKeyedServices(Collections.VaultsKey)]
-        IMongoCollection<SyncItem> vaults,
-        [FromKeyedServices(Collections.TagsKey)]
-        IMongoCollection<SyncItem> tags,
-
-        Repository<UserSettings> usersSettings, Repository<Monograph> monographs)
+        public SyncItemsRepositoryAccessor(SyncItemsRepository<Note> _notes,
+        SyncItemsRepository<Notebook> _notebooks,
+        SyncItemsRepository<Content> _content,
+        SyncItemsRepository<Setting> _settings,
+        SyncItemsRepository<Attachment> _attachments,
+        SyncItemsRepository<Shortcut> _shortcuts,
+        SyncItemsRepository<Relation> _relations,
+        SyncItemsRepository<Reminder> _reminders,
+        Repository<UserSettings> _usersSettings,
+        Repository<Monograph> _monographs)
         {
-            UsersSettings = usersSettings;
-            Monographs = monographs;
-            Notebooks = new SyncItemsRepository(dbContext, notebooks);
-            Notes = new SyncItemsRepository(dbContext, notes);
-            Contents = new SyncItemsRepository(dbContext, content);
-            Settings = new SyncItemsRepository(dbContext, settings);
-            LegacySettings = new SyncItemsRepository(dbContext, legacySettings);
-            Attachments = new SyncItemsRepository(dbContext, attachments);
-            Shortcuts = new SyncItemsRepository(dbContext, shortcuts);
-            Reminders = new SyncItemsRepository(dbContext, reminders);
-            Relations = new SyncItemsRepository(dbContext, relations);
-            Colors = new SyncItemsRepository(dbContext, colors);
-            Vaults = new SyncItemsRepository(dbContext, vaults);
-            Tags = new SyncItemsRepository(dbContext, tags);
+            Notebooks = _notebooks;
+            Notes = _notes;
+            Contents = _content;
+            Settings = _settings;
+            Attachments = _attachments;
+            UsersSettings = _usersSettings;
+            Monographs = _monographs;
+            Shortcuts = _shortcuts;
+            Reminders = _reminders;
+            Relations = _relations;
         }
     }
 }

Notesnook.API/Authorization/EmailVerifiedRequirement.cs

@@ -0,0 +1,36 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+
+namespace Notesnook.API.Authorization
+{
+    public class EmailVerifiedRequirement : AuthorizationHandler<EmailVerifiedRequirement>, IAuthorizationRequirement
+    {
+        protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, EmailVerifiedRequirement requirement)
+        {
+            var isEmailVerified = context.User.HasClaim("verified", "true");
+            var isUserBasic = context.User.HasClaim("notesnook:status", "basic") || context.User.HasClaim("notesnook:status", "premium_expired");
+            if (!isUserBasic || isEmailVerified)
+                context.Succeed(requirement);
+            return Task.CompletedTask;
+        }
+    }
+}

Notesnook.API/Authorization/ProUserRequirement.cs

@@ -17,47 +17,21 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
-using System.Security.Claims;
-using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
 
 namespace Notesnook.API.Authorization
 {
     public class ProUserRequirement : AuthorizationHandler<ProUserRequirement>, IAuthorizationRequirement
     {
-        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
-        {
-            ["/s3"] = "upload attachments",
-            ["/s3/multipart"] = "upload attachments",
-        };
-        private readonly string[] allowedClaims = ["trial", "premium", "premium_canceled"];
+        private string[] allowedClaims = { "trial", "premium", "premium_canceled" };
         protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, ProUserRequirement requirement)
         {
-            PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
-            var isProOrTrial = context.User.Claims.Any((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
-            if (isProOrTrial) context.Succeed(requirement);
-            else
-            {
-                var phrase = "continue";
-                foreach (var item in pathErrorPhraseMap)
-                {
-                    if (path != null && path.StartsWithSegments(item.Key))
-                        phrase = item.Value;
-                }
-                var error = $"Please upgrade to Pro to {phrase}.";
-                context.Fail(new AuthorizationFailureReason(this, error));
-            }
+            var isProOrTrial = context.User.HasClaim((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
+            if (isProOrTrial)
+                context.Succeed(requirement);
             return Task.CompletedTask;
         }
-
-        public override Task HandleAsync(AuthorizationHandlerContext context)
-        {
-            return this.HandleRequirementAsync(context, this);
-        }
     }
 }

Notesnook.API/Authorization/SyncRequirement.cs

@@ -29,23 +29,27 @@ namespace Notesnook.API.Authorization
 {
     public class SyncRequirement : AuthorizationHandler<SyncRequirement>, IAuthorizationRequirement
     {
-        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
+        private Dictionary<string, string> pathErrorPhraseMap = new Dictionary<string, string>
         {
             ["/sync/attachments"] = "use attachments",
             ["/sync"] = "sync your notes",
             ["/hubs/sync"] = "sync your notes",
-            ["/hubs/sync/v2"] = "sync your notes",
             ["/monographs"] = "publish monographs"
         };
 
+        private string[] allowedClaims = { "trial", "premium", "premium_canceled" };
         protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, SyncRequirement requirement)
         {
             PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
             var result = this.IsAuthorized(context.User, path);
             if (result.Succeeded) context.Succeed(requirement);
-            else if (result.AuthorizationFailure.FailureReasons.Any())
-                context.Fail(result.AuthorizationFailure.FailureReasons.First());
-            else context.Fail();
+            else
+            {
+                var hasReason = result.AuthorizationFailure.FailureReasons.Count() > 0;
+                if (hasReason)
+                    context.Fail(result.AuthorizationFailure.FailureReasons.First());
+                else context.Fail();
+            }
 
             return Task.CompletedTask;
         }
@@ -56,7 +60,7 @@ namespace Notesnook.API.Authorization
 
             if (string.IsNullOrEmpty(id))
             {
-                var reason = new[]
+                var reason = new AuthorizationFailureReason[]
                 {
                     new AuthorizationFailureReason(this, "Invalid token.")
                 };
@@ -80,7 +84,7 @@ namespace Notesnook.API.Authorization
                 }
 
                 var error = $"Please confirm your email to {phrase}.";
-                var reason = new[]
+                var reason = new AuthorizationFailureReason[]
                 {
                     new AuthorizationFailureReason(this, error)
                 };
@@ -88,6 +92,7 @@ namespace Notesnook.API.Authorization
                 //  context.Fail(new AuthorizationFailureReason(this, error));
             }
 
+            var isProOrTrial = User.HasClaim((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
             if (hasSyncScope && isInAudience && hasRole && isEmailVerified)
                 return PolicyAuthorizationResult.Success(); //(requirement);
             return PolicyAuthorizationResult.Forbid();

Notesnook.API/Constants.cs

@@ -1,18 +0,0 @@
-namespace Notesnook.API
-{
-    public class Collections
-    {
-        public const string SettingsKey = "settingsv2";
-        public const string AttachmentsKey = "attachments";
-        public const string ContentKey = "content";
-        public const string NotesKey = "notes";
-        public const string NotebooksKey = "notebooks";
-        public const string RelationsKey = "relations";
-        public const string RemindersKey = "reminders";
-        public const string LegacySettingsKey = "settings";
-        public const string ShortcutsKey = "shortcuts";
-        public const string TagsKey = "tags";
-        public const string ColorsKey = "colors";
-        public const string VaultsKey = "vaults";
-    }
-}

Notesnook.API/Controllers/AnnouncementController.cs

@@ -18,12 +18,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using MongoDB.Driver;
 using Notesnook.API.Models;
 using Streetwriters.Data.Repositories;
 
@@ -44,26 +42,10 @@ namespace Notesnook.API.Controllers
         [AllowAnonymous]
         public async Task<IActionResult> GetActiveAnnouncements([FromQuery] string userId)
         {
-            var totalActive = await Announcements.Collection.CountDocumentsAsync(Builders<Announcement>.Filter.Eq("IsActive", true));
-            if (totalActive <= 0) return Ok(new Announcement[] { });
-
-            var announcements = (await Announcements.FindAsync((a) => a.IsActive)).Where((a) => a.UserIds == null || a.UserIds.Length == 0 || a.UserIds.Contains(userId));
-            foreach (var announcement in announcements)
-            {
-                if (announcement.UserIds != null && !announcement.UserIds.Contains(userId)) continue;
-
-                foreach (var item in announcement.Body)
-                {
-                    if (item.Type != "callToActions") continue;
-                    foreach (var action in item.Actions)
-                    {
-                        if (action.Type != "link") continue;
-
-                        action.Data = action.Data.Replace("{{UserId}}", userId ?? "0");
-                    }
-                }
-            }
-            return Ok(announcements);
+            var announcements = await Announcements.FindAsync((a) => a.IsActive);
+            return Ok(announcements.Where((a) => a.UserIds != null && a.UserIds.Length > 0
+                                                    ? a.UserIds.Contains(userId)
+                                                    : true));
         }
     }
 }

Notesnook.API/Controllers/MonographsController.cs

@@ -18,19 +18,12 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
-using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using MongoDB.Bson;
-using MongoDB.Driver;
 using Notesnook.API.Models;
-using Notesnook.API.Services;
-using Streetwriters.Common;
-using Streetwriters.Common.Messages;
 using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
 
@@ -41,7 +34,6 @@ namespace Notesnook.API.Controllers
     [Authorize("Sync")]
     public class MonographsController : ControllerBase
     {
-        const string SVG_PIXEL = "<svg xmlns='http://www.w3.org/2000/svg' width='1' height='1'><circle r='9'/></svg>";
         private Repository<Monograph> Monographs { get; set; }
         private readonly IUnitOfWork unit;
         private const int MAX_DOC_SIZE = 15 * 1024 * 1024;
@@ -51,145 +43,50 @@ namespace Notesnook.API.Controllers
             unit = unitOfWork;
         }
 
-        private static FilterDefinition<Monograph> CreateMonographFilter(string userId, Monograph monograph)
-        {
-            var userIdFilter = Builders<Monograph>.Filter.Eq("UserId", userId);
-            return ObjectId.TryParse(monograph.ItemId, out ObjectId id)
-            ? Builders<Monograph>.Filter
-                .And(userIdFilter,
-                    Builders<Monograph>.Filter.Or(
-                        Builders<Monograph>.Filter.Eq("_id", id), Builders<Monograph>.Filter.Eq("ItemId", monograph.ItemId)
-                    )
-                )
-            : Builders<Monograph>.Filter
-                .And(userIdFilter,
-                    Builders<Monograph>.Filter.Eq("ItemId", monograph.ItemId)
-                );
-        }
-
-        private static FilterDefinition<Monograph> CreateMonographFilter(string itemId)
-        {
-            return ObjectId.TryParse(itemId, out ObjectId id)
-            ? Builders<Monograph>.Filter.Or(
-                Builders<Monograph>.Filter.Eq("_id", id),
-                Builders<Monograph>.Filter.Eq("ItemId", itemId))
-            : Builders<Monograph>.Filter.Eq("ItemId", itemId);
-        }
-
-        private async Task<Monograph> FindMonographAsync(string userId, Monograph monograph)
-        {
-            var result = await Monographs.Collection.FindAsync(CreateMonographFilter(userId, monograph), new FindOptions<Monograph>
-            {
-                Limit = 1
-            });
-            return await result.FirstOrDefaultAsync();
-        }
-
-        private async Task<Monograph> FindMonographAsync(string itemId)
-        {
-            var result = await Monographs.Collection.FindAsync(CreateMonographFilter(itemId), new FindOptions<Monograph>
-            {
-                Limit = 1
-            });
-            return await result.FirstOrDefaultAsync();
-        }
-
         [HttpPost]
-        public async Task<IActionResult> PublishAsync([FromQuery] string deviceId, [FromBody] Monograph monograph)
+        public async Task<IActionResult> PublishAsync([FromBody] Monograph monograph)
         {
-            try
+            var userId = this.User.FindFirstValue("sub");
+            if (userId == null) return Unauthorized();
+
+            if (await Monographs.GetAsync(monograph.Id) != null) return base.Conflict("This monograph is already published.");
+
+            if (monograph.EncryptedContent == null)
+                monograph.CompressedContent = monograph.Content.CompressBrotli();
+            monograph.UserId = userId;
+            monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+
+
+            if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
+                return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
+
+            Monographs.Insert(monograph);
+
+            if (!await unit.Commit()) return BadRequest();
+            return Ok(new
             {
-                var userId = this.User.FindFirstValue("sub");
-                if (userId == null) return Unauthorized();
-
-                var existingMonograph = await FindMonographAsync(userId, monograph);
-                if (existingMonograph != null && !existingMonograph.Deleted)
-                {
-                    return base.Conflict("This monograph is already published.");
-                }
-
-                if (monograph.EncryptedContent == null)
-                    monograph.CompressedContent = monograph.Content.CompressBrotli();
-                monograph.UserId = userId;
-                monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-
-                if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
-                    return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
-
-                if (existingMonograph != null)
-                {
-                    monograph.Id = existingMonograph?.Id;
-                }
-                monograph.Deleted = false;
-                await Monographs.Collection.ReplaceOneAsync(
-                    CreateMonographFilter(userId, monograph),
-                    monograph,
-                    new ReplaceOptions { IsUpsert = true }
-                );
-
-                await MarkMonographForSyncAsync(monograph.ItemId ?? monograph.Id, deviceId);
-
-                return Ok(new
-                {
-                    id = monograph.ItemId,
-                    datePublished = monograph.DatePublished,
-                });
-            }
-            catch (Exception e)
-            {
-                await Slogger<MonographsController>.Error(nameof(PublishAsync), e.ToString());
-                return BadRequest();
-            }
+                id = monograph.Id
+            });
         }
 
         [HttpPatch]
-        public async Task<IActionResult> UpdateAsync([FromQuery] string deviceId, [FromBody] Monograph monograph)
+        public async Task<IActionResult> UpdateAsync([FromBody] Monograph monograph)
         {
-            try
+            if (await Monographs.GetAsync(monograph.Id) == null) return NotFound();
+
+            if (monograph.EncryptedContent == null)
+                monograph.CompressedContent = monograph.Content.CompressBrotli();
+            else
+                monograph.Content = null;
+
+            monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+            Monographs.Update(monograph.Id, monograph);
+
+            if (!await unit.Commit()) return BadRequest();
+            return Ok(new
             {
-                var userId = this.User.FindFirstValue("sub");
-                if (userId == null) return Unauthorized();
-
-                var existingMonograph = await FindMonographAsync(userId, monograph);
-                if (existingMonograph == null || existingMonograph.Deleted)
-                {
-                    return NotFound();
-                }
-
-                if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
-                    return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
-
-                if (monograph.EncryptedContent == null)
-                    monograph.CompressedContent = monograph.Content.CompressBrotli();
-                else
-                    monograph.Content = null;
-
-                monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                var result = await Monographs.Collection.UpdateOneAsync(
-                    CreateMonographFilter(userId, monograph),
-                    Builders<Monograph>.Update
-                    .Set(m => m.DatePublished, monograph.DatePublished)
-                    .Set(m => m.CompressedContent, monograph.CompressedContent)
-                    .Set(m => m.EncryptedContent, monograph.EncryptedContent)
-                    .Set(m => m.SelfDestruct, monograph.SelfDestruct)
-                    .Set(m => m.Title, monograph.Title)
-                    .Set(m => m.Password, monograph.Password)
-                );
-                if (!result.IsAcknowledged) return BadRequest();
-
-                await MarkMonographForSyncAsync(monograph.ItemId ?? monograph.Id, deviceId);
-
-                return Ok(new
-                {
-                    id = monograph.ItemId,
-                    datePublished = monograph.DatePublished,
-                });
-            }
-            catch (Exception e)
-            {
-                await Slogger<MonographsController>.Error(nameof(UpdateAsync), e.ToString());
-                return BadRequest();
-            }
+                id = monograph.Id
+            });
         }
 
         [HttpGet]
@@ -198,24 +95,17 @@ namespace Notesnook.API.Controllers
             var userId = this.User.FindFirstValue("sub");
             if (userId == null) return Unauthorized();
 
-            var monographs = (await Monographs.Collection.FindAsync(
-                    Builders<Monograph>.Filter.And(
-                        Builders<Monograph>.Filter.Eq("UserId", userId),
-                        Builders<Monograph>.Filter.Eq("Deleted", false)
-                    )
-               , new FindOptions<Monograph, ObjectWithId>
-               {
-                   Projection = Builders<Monograph>.Projection.Include("_id").Include("ItemId"),
-               })).ToEnumerable();
-            return Ok(monographs.Select((m) => m.ItemId ?? m.Id));
+            var userMonographs = await Monographs.FindAsync((m) => m.UserId == userId);
+            return Ok(userMonographs.Select((m) => m.Id));
         }
 
+
         [HttpGet("{id}")]
         [AllowAnonymous]
         public async Task<IActionResult> GetMonographAsync([FromRoute] string id)
         {
-            var monograph = await FindMonographAsync(id);
-            if (monograph == null || monograph.Deleted)
+            var monograph = await Monographs.FindOneAsync((m) => m.Id == id);
+            if (monograph == null)
             {
                 return NotFound(new
                 {
@@ -224,100 +114,21 @@ namespace Notesnook.API.Controllers
                 });
             }
 
+            if (monograph.SelfDestruct)
+                await Monographs.DeleteByIdAsync(monograph.Id);
+
             if (monograph.EncryptedContent == null)
                 monograph.Content = monograph.CompressedContent.DecompressBrotli();
-            if (monograph.ItemId == null) monograph.ItemId = monograph.Id;
             return Ok(monograph);
         }
 
-        [HttpGet("{id}/view")]
-        [AllowAnonymous]
-        public async Task<IActionResult> TrackView([FromRoute] string id)
-        {
-            var monograph = await FindMonographAsync(id);
-            if (monograph == null || monograph.Deleted) return Content(SVG_PIXEL, "image/svg+xml");
-
-            if (monograph.SelfDestruct)
-            {
-                var userId = this.User.FindFirstValue("sub");
-                await Monographs.Collection.ReplaceOneAsync(
-                    CreateMonographFilter(userId, monograph),
-                    new Monograph
-                    {
-                        ItemId = id,
-                        Id = monograph.Id,
-                        Deleted = true
-                    }
-                );
-
-                await MarkMonographForSyncAsync(id);
-            }
-
-            return Content(SVG_PIXEL, "image/svg+xml");
-        }
 
         [HttpDelete("{id}")]
-        public async Task<IActionResult> DeleteAsync([FromQuery] string deviceId, [FromRoute] string id)
+        public async Task<IActionResult> DeleteAsync([FromRoute] string id)
         {
-            var monograph = await FindMonographAsync(id);
-            if (monograph == null || monograph.Deleted)
-            {
-                return NotFound(new
-                {
-                    error = "invalid_id",
-                    error_description = $"No such monograph found."
-                });
-            }
-
-            var userId = this.User.FindFirstValue("sub");
-            await Monographs.Collection.ReplaceOneAsync(
-                CreateMonographFilter(userId, monograph),
-                new Monograph
-                {
-                    ItemId = id,
-                    Id = monograph.Id,
-                    Deleted = true,
-                    UserId = monograph.UserId
-                }
-            );
-
-            await MarkMonographForSyncAsync(id, deviceId);
-
+            Monographs.DeleteById(id);
+            if (!await unit.Commit()) return BadRequest();
             return Ok();
         }
-
-        private async Task MarkMonographForSyncAsync(string monographId, string deviceId)
-        {
-            if (deviceId == null) return;
-            var userId = this.User.FindFirstValue("sub");
-
-            new SyncDeviceService(new SyncDevice(userId, deviceId)).AddIdsToOtherDevices([$"{monographId}:monograph"]);
-            await SendTriggerSyncEventAsync();
-        }
-
-        private async Task MarkMonographForSyncAsync(string monographId)
-        {
-            var userId = this.User.FindFirstValue("sub");
-
-            new SyncDeviceService(new SyncDevice(userId, string.Empty)).AddIdsToAllDevices([$"{monographId}:monograph"]);
-            await SendTriggerSyncEventAsync(sendToAllDevices: true);
-        }
-
-        private async Task SendTriggerSyncEventAsync(bool sendToAllDevices = false)
-        {
-            var userId = this.User.FindFirstValue("sub");
-            var jti = this.User.FindFirstValue("jti");
-
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
-            {
-                OriginTokenId = sendToAllDevices ? null : jti,
-                UserId = userId,
-                Message = new Message
-                {
-                    Type = "triggerSync",
-                    Data = JsonSerializer.Serialize(new { reason = "Monographs updated." })
-                }
-            });
-        }
     }
 }

Notesnook.API/Controllers/S3Controller.cs

@@ -24,12 +24,12 @@ using System.Threading.Tasks;
 using System.Security.Claims;
 using Notesnook.API.Interfaces;
 using System;
-using Notesnook.API.Models;
 
 namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Route("s3")]
+    [Authorize("Sync")]
     [ResponseCache(NoStore = true, Location = ResponseCacheLocation.None)]
     public class S3Controller : ControllerBase
     {
@@ -40,7 +40,6 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPut]
-        [Authorize("Pro")]
         public IActionResult Upload([FromQuery] string name)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -51,7 +50,6 @@ namespace Notesnook.API.Controllers
 
 
         [HttpGet("multipart")]
-        [Authorize("Pro")]
         public async Task<IActionResult> MultipartUpload([FromQuery] string name, [FromQuery] int parts, [FromQuery] string uploadId)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -64,7 +62,6 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpDelete("multipart")]
-        [Authorize("Pro")]
         public async Task<IActionResult> AbortMultipartUpload([FromQuery] string name, [FromQuery] string uploadId)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -77,20 +74,19 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPost("multipart")]
-        [Authorize("Pro")]
-        public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequestWrapper uploadRequestWrapper)
+        public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequest uploadRequest)
         {
             var userId = this.User.FindFirstValue("sub");
             try
             {
-                await S3Service.CompleteMultipartUploadAsync(userId, uploadRequestWrapper.ToRequest());
+                await S3Service.CompleteMultipartUploadAsync(userId, uploadRequest);
                 return Ok();
             }
             catch (Exception ex) { return BadRequest(ex.Message); }
         }
 
         [HttpGet]
-        [Authorize("Sync")]
+        [Authorize]
         public IActionResult Download([FromQuery] string name)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -100,17 +96,18 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpHead]
-        [Authorize("Sync")]
+        [Authorize]
         public async Task<IActionResult> Info([FromQuery] string name)
         {
             var userId = this.User.FindFirstValue("sub");
             var size = await S3Service.GetObjectSizeAsync(userId, name);
+            if (size == null) return BadRequest();
+
             HttpContext.Response.Headers.ContentLength = size;
             return Ok();
         }
 
         [HttpDelete]
-        [Authorize("Sync")]
         public async Task<IActionResult> DeleteAsync([FromQuery] string name)
         {
             try

Notesnook.API/Controllers/SyncDeviceController.cs

@@ -1,74 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Net.Http;
-using System.Security.Claims;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.Mvc;
-using Notesnook.API.Interfaces;
-using Notesnook.API.Models.Responses;
-using Notesnook.API.Services;
-using Streetwriters.Common;
-using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Models;
-
-namespace Notesnook.API.Controllers
-{
-    [ApiController]
-    [Authorize]
-    [Route("devices")]
-    public class SyncDeviceController : ControllerBase
-    {
-        [HttpPost]
-        public async Task<IActionResult> RegisterDevice([FromQuery] string deviceId)
-        {
-            try
-            {
-                var userId = this.User.FindFirstValue("sub") ?? throw new Exception("User not found.");
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).RegisterDevice();
-                return Ok();
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't register device.", ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
-        }
-
-
-        [HttpDelete]
-        public async Task<IActionResult> UnregisterDevice([FromQuery] string deviceId)
-        {
-            try
-            {
-                var userId = this.User.FindFirstValue("sub") ?? throw new Exception("User not found.");
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).UnregisterDevice();
-                return Ok();
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't unregister device.", ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
-        }
-    }
-}

Notesnook.API/Controllers/UsersController.cs

@@ -18,23 +18,35 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
+using System.Net.Http;
 using System.Security.Claims;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http.Timeouts;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Notesnook.API.Interfaces;
-using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
 using Streetwriters.Common;
+using Streetwriters.Common.Extensions;
+using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Authorize]
     [Route("users")]
-    public class UsersController(IUserService UserService) : ControllerBase
+    public class UsersController : ControllerBase
     {
+        private readonly HttpClient httpClient;
+        private readonly IHttpContextAccessor HttpContextAccessor;
+        private IUserService UserService { get; set; }
+        public UsersController(IUserService userService, IHttpContextAccessor accessor)
+        {
+            httpClient = new HttpClient();
+            HttpContextAccessor = accessor;
+            UserService = userService;
+        }
+
         [HttpPost]
         [AllowAnonymous]
         public async Task<IActionResult> Signup()
@@ -54,39 +66,21 @@ namespace Notesnook.API.Controllers
         [HttpGet]
         public async Task<IActionResult> GetUser()
         {
-            var userId = User.FindFirstValue("sub");
-            try
-            {
-                UserResponse response = await UserService.GetUserAsync(userId);
-                if (!response.Success) return BadRequest(response);
-                return Ok(response);
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't get user for id.", userId, ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
+            UserResponse response = await UserService.GetUserAsync();
+            if (!response.Success) return BadRequest(response);
+            return Ok(response);
         }
 
         [HttpPatch]
         public async Task<IActionResult> UpdateUser([FromBody] UserResponse user)
         {
-            var userId = User.FindFirstValue("sub");
-            try
-            {
-                var keys = new UserKeys
-                {
-                    AttachmentsKey = user.AttachmentsKey,
-                    MonographPasswordsKey = user.MonographPasswordsKey
-                };
-                await UserService.SetUserKeysAsync(userId, keys);
-                return Ok();
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't update user with id.", userId, ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
+            UserResponse response = await UserService.GetUserAsync(false);
+
+            if (user.AttachmentsKey != null)
+                await UserService.SetUserAttachmentsKeyAsync(response.UserId, user.AttachmentsKey);
+            else return BadRequest();
+
+            return Ok();
         }
 
         [HttpPost("reset")]
@@ -100,20 +94,24 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPost("delete")]
-        [RequestTimeout(5 * 60 * 1000)]
-        public async Task<IActionResult> Delete([FromForm] DeleteAccountForm form)
+        public async Task<IActionResult> Delete()
         {
-            var userId = this.User.FindFirstValue("sub");
-            var jti = User.FindFirstValue("jti");
             try
             {
-                await UserService.DeleteUserAsync(userId, jti, form.Password);
-                return Ok();
+                var userId = this.User.FindFirstValue("sub");
+
+                if (await UserService.DeleteUserAsync(userId, User.FindFirstValue("jti")))
+                {
+                    Response response = await this.httpClient.ForwardAsync<Response>(this.HttpContextAccessor, $"{Servers.IdentityServer.ToString()}/account/unregister", HttpMethod.Post);
+                    if (!response.Success) return BadRequest();
+
+                    return Ok();
+                }
+                return BadRequest();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't delete user with id.", userId, ex.ToString());
-                return BadRequest(new { error = ex.Message });
+                return BadRequest(ex.Message);
             }
         }
     }

Notesnook.API/Dockerfile

@@ -1,50 +1,28 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build
 WORKDIR /app
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
-ARG TARGETARCH
-ARG BUILDPLATFORM
-ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
-
-WORKDIR /src
-
+# restore all project dependencies
 COPY Streetwriters.Data/*.csproj ./Streetwriters.Data/
+RUN dotnet restore /app/Streetwriters.Data/Streetwriters.Data.csproj --use-current-runtime
+
 COPY Streetwriters.Common/*.csproj ./Streetwriters.Common/
+RUN dotnet restore /app/Streetwriters.Common/Streetwriters.Common.csproj --use-current-runtime
+
 COPY Notesnook.API/*.csproj ./Notesnook.API/
+RUN dotnet restore /app/Notesnook.API/Notesnook.API.csproj --use-current-runtime
 
-# restore dependencies
-RUN dotnet restore -v d /src/Notesnook.API/Notesnook.API.csproj --use-current-runtime
-
+# copy everything else
 COPY Streetwriters.Data/ ./Streetwriters.Data/
 COPY Streetwriters.Common/ ./Streetwriters.Common/
 COPY Notesnook.API/ ./Notesnook.API/
 
-WORKDIR /src/Notesnook.API/
+# build
+WORKDIR /app/Notesnook.API/
+ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
+RUN dotnet publish -c Release -o /app/out --use-current-runtime --self-contained false --no-restore
 
-RUN dotnet build -c Release -o /app/build -a $TARGETARCH
-
-FROM build AS publish
-RUN dotnet publish -c Release -o /app/publish \
-  #--runtime alpine-x64 \
-  --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
-  /p:PublishSingleFile=true \
-  /p:JsonSerializerIsReflectionEnabledByDefault=true \
-  -a $TARGETARCH
-
-FROM --platform=$BUILDPLATFORM base AS final
-ARG TARGETARCH
-ARG BUILDPLATFORM
-
-# create a new user and change directory ownership
-RUN adduser --disabled-password \
-  --home /app \
-  --gecos '' dotnetuser && chown -R dotnetuser /app
-
-# impersonate into the new user
-USER dotnetuser
+# final stage/image
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
 WORKDIR /app
-
-COPY --from=publish /app/publish .
-ENTRYPOINT ["./Notesnook.API"]
+COPY --from=build /app/out .
+ENTRYPOINT ["dotnet", "Notesnook.API.dll"]

Notesnook.API/EventSources/SyncEventCounterSource.cs

@@ -1,49 +0,0 @@
-using System;
-using System.Diagnostics.Metrics;
-using System.Diagnostics.Tracing;
-
-[EventSource(Name = "Notesnook.API.EventCounter.Sync")]
-public sealed class SyncEventCounterSource : EventSource
-{
-    public static readonly SyncEventCounterSource Log = new();
-
-    private Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
-    private Counter<int> fetchCounter;
-    private Counter<int> pushCounter;
-    private Counter<int> legacyFetchCounter;
-    private Counter<int> pushV2Counter;
-    private Counter<int> fetchV2Counter;
-    private Histogram<long> fetchV2Duration;
-    private Histogram<long> pushV2Duration;
-    private SyncEventCounterSource()
-    {
-        fetchCounter = meter.CreateCounter<int>("sync.fetches", "fetches", "Total fetches");
-        pushCounter = meter.CreateCounter<int>("sync.pushes", "pushes", "Total pushes");
-        legacyFetchCounter = meter.CreateCounter<int>("sync.legacy-fetches", "fetches", "Total legacy fetches");
-        fetchV2Counter = meter.CreateCounter<int>("sync.v2.fetches", "fetches", "Total v2 fetches");
-        pushV2Counter = meter.CreateCounter<int>("sync.v2.pushes", "pushes", "Total v2 pushes");
-        fetchV2Duration = meter.CreateHistogram<long>("sync.v2.fetch_duration");
-        pushV2Duration = meter.CreateHistogram<long>("sync.v2.push_duration");
-    }
-
-    public void Fetch() => fetchCounter.Add(1);
-    public void LegacyFetch() => legacyFetchCounter.Add(1);
-    public void FetchV2() => fetchV2Counter.Add(1);
-    public void PushV2() => pushV2Counter.Add(1);
-    public void Push() => pushCounter.Add(1);
-    public void RecordFetchDuration(long durationMs) => fetchV2Duration.Record(durationMs);
-    public void RecordPushDuration(long durationMs) => pushV2Duration.Record(durationMs);
-
-    protected override void Dispose(bool disposing)
-    {
-        legacyFetchCounter = null;
-        fetchV2Counter = null;
-        pushV2Counter = null;
-        pushCounter = null;
-        fetchCounter = null;
-        meter.Dispose();
-        meter = null;
-
-        base.Dispose(disposing);
-    }
-}

Notesnook.API/Extensions/AuthorizationResultTransformer.cs

@@ -48,7 +48,7 @@ namespace Notesnook.API.Extensions
             {
                 var error = string.Join("\n", policyAuthorizationResult.AuthorizationFailure.FailureReasons.Select((r) => r.Message));
 
-                if (!string.IsNullOrEmpty(error))
+                if (!string.IsNullOrEmpty(error) && !isWebsocket)
                 {
                     httpContext.Response.StatusCode = (int)HttpStatusCode.Unauthorized;
                     httpContext.Response.ContentType = "application/json";

Notesnook.API/Hubs/SyncHub.cs

@@ -23,114 +23,24 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Security.Claims;
 using System.Text.Json;
-using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.SignalR;
-using MongoDB.Driver;
 using Notesnook.API.Authorization;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
-using Notesnook.API.Repositories;
 using Streetwriters.Common.Models;
 using Streetwriters.Data.Interfaces;
 
 namespace Notesnook.API.Hubs
 {
-    public struct RunningPush
-    {
-        public long Timestamp { get; set; }
-        public long Validity { get; set; }
-        public string ConnectionId { get; set; }
-    }
     public interface ISyncHubClient
     {
-        Task PushItems(SyncTransferItemV2 transferItem);
-        Task<bool> SendItems(SyncTransferItemV2 transferItem);
-        Task PushCompleted(long lastSynced);
-    }
-
-    public class GlobalSync
-    {
-        private const long PUSH_VALIDITY_EXTENSION_PERIOD = 16 * 1000; // 16 second
-        private const int PUSH_VALIDITY_PERIOD_PER_ITEM = 5 * 100; // 0.5 second
-        private const long BASE_PUSH_VALIDITY_PERIOD = 5 * 1000; // 5 seconds
-        private const long BASE_PUSH_VALIDITY_PERIOD_NEW = 16 * 1000; // 16 seconds
-        private readonly static Dictionary<string, List<RunningPush>> PushOperations = new();
-
-        public static void ClearPushOperations(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations.ToArray())
-                    if (push.ConnectionId == connectionId || !IsPushValid(push, now))
-                        operations.Remove(push);
-            }
-        }
-
-        public static bool IsPushing(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (push.ConnectionId == connectionId && IsPushValid(push, now)) return true;
-            }
-            return false;
-        }
-
-
-        public static bool IsUserPushing(string userId)
-        {
-            var count = 0;
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (IsPushValid(push, now)) ++count;
-            }
-            return count > 0;
-        }
-
-        public static void StartPush(string userId, string connectionId, long? totalItems = null)
-        {
-            if (IsPushing(userId, connectionId)) return;
-
-            if (!PushOperations.ContainsKey(userId))
-                PushOperations[userId] = new List<RunningPush>();
-
-            PushOperations[userId].Add(new RunningPush
-            {
-                ConnectionId = connectionId,
-                Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-                Validity = totalItems.HasValue ? BASE_PUSH_VALIDITY_PERIOD + (totalItems.Value * PUSH_VALIDITY_PERIOD_PER_ITEM) : BASE_PUSH_VALIDITY_PERIOD_NEW
-            });
-        }
-        public static void ExtendPush(string userId, string connectionId)
-        {
-            if (!IsPushing(userId, connectionId) || !PushOperations.ContainsKey(userId))
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var index = PushOperations[userId].FindIndex((push) => push.ConnectionId == connectionId);
-            if (index < 0)
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var pushOperation = PushOperations[userId][index];
-            pushOperation.Validity += PUSH_VALIDITY_EXTENSION_PERIOD;
-        }
-        private static bool IsPushValid(RunningPush push, long now)
-        {
-            return now < push.Timestamp + push.Validity;
-        }
+        Task SyncItem(SyncTransferItem transferItem);
+        Task RemoteSyncCompleted(long lastSynced);
+        Task SyncCompleted();
     }
 
     [Authorize("Sync")]
@@ -138,16 +48,6 @@ namespace Notesnook.API.Hubs
     {
         private ISyncItemsRepositoryAccessor Repositories { get; }
         private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = new[] {
-            "settings",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "relation", // relations must sync at the end to prevent invalid state
-        };
 
         public SyncHub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
         {
@@ -170,235 +70,181 @@ namespace Notesnook.API.Hubs
 
         public override async Task OnDisconnectedAsync(Exception exception)
         {
-            try
-            {
-                await base.OnDisconnectedAsync(exception);
-            }
-            finally
-            {
-                var id = Context.User.FindFirstValue("sub");
-                GlobalSync.ClearPushOperations(id, Context.ConnectionId);
-            }
+            var id = Context.User.FindFirstValue("sub");
+            await Groups.RemoveFromGroupAsync(Context.ConnectionId, id);
+            await base.OnDisconnectedAsync(exception);
         }
 
-        private Action<SyncItem, string, long> MapTypeToUpsertAction(string type)
+        public async Task<int> SyncItem(BatchedSyncTransferItem transferItem)
         {
-            return type switch
-            {
-                "attachment" => Repositories.Attachments.Upsert,
-                "note" => Repositories.Notes.Upsert,
-                "notebook" => Repositories.Notebooks.Upsert,
-                "content" => Repositories.Contents.Upsert,
-                "shortcut" => Repositories.Shortcuts.Upsert,
-                "reminder" => Repositories.Reminders.Upsert,
-                "relation" => Repositories.Relations.Upsert,
-                _ => null,
-            };
-        }
-
-        public async Task<long> InitializePush(SyncMetadata syncMetadata)
-        {
-            if (syncMetadata.LastSynced <= 0) throw new HubException("Last synced time cannot be zero or less than zero.");
 
             var userId = Context.User.FindFirstValue("sub");
             if (string.IsNullOrEmpty(userId)) return 0;
 
-            UserSettings userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            long dateSynced = Math.Max(syncMetadata.LastSynced, userSettings.LastSynced);
+            var others = Clients.OthersInGroup(userId);
 
-            GlobalSync.StartPush(userId, Context.ConnectionId);
+            UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
 
+            long dateSynced = transferItem.LastSynced > userSettings.LastSynced ? transferItem.LastSynced : userSettings.LastSynced;
 
-            if (
-                (userSettings.VaultKey != null &&
-                syncMetadata.VaultKey != null &&
-                !userSettings.VaultKey.Equals(syncMetadata.VaultKey) &&
-                !syncMetadata.VaultKey.IsEmpty()) ||
-                (userSettings.VaultKey == null &&
-                syncMetadata.VaultKey != null &&
-                !syncMetadata.VaultKey.IsEmpty()))
+            Parallel.For(0, transferItem.Items.Length, async (i) =>
             {
-                userSettings.VaultKey = syncMetadata.VaultKey;
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
-            }
+                var data = transferItem.Items[i];
+                var type = transferItem.Types[i];
+                var id = transferItem.Ids[i];
 
-            return dateSynced;
-        }
+                // We intentionally don't await here to speed up the sync. Fire and forget
+                // suits here because we don't really care if the item reaches the other
+                // devices.
+                others.SyncItem(
+                        new SyncTransferItem
+                        {
+                            Item = data,
+                            ItemType = type,
+                            LastSynced = dateSynced,
+                            Total = transferItem.Total,
+                            Current = transferItem.Current + i
+                        });
 
-        public async Task<int> PushItems(SyncTransferItemV2 pushItem, long dateSynced)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) return 0;
-
-            SyncEventCounterSource.Log.Push();
-
-            try
-            {
-                var others = Clients.OthersInGroup(userId);
-                others.PushItems(pushItem);
-
-                GlobalSync.ExtendPush(userId, Context.ConnectionId);
-
-                if (pushItem.Type == "settings")
+                switch (type)
                 {
-                    var settings = pushItem.Items.First();
-                    if (settings == null) return 0;
-                    settings.Id = MongoDB.Bson.ObjectId.Parse(userId);
-                    settings.ItemId = userId;
-                    Repositories.LegacySettings.Upsert(settings, userId, dateSynced);
-                }
-                else
-                {
-                    var UpsertItem = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception("Invalid item type.");
-                    foreach (var item in pushItem.Items)
-                    {
-                        UpsertItem(item, userId, dateSynced);
-                    }
+                    case "content":
+                        await Repositories.Contents.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "attachment":
+                        await Repositories.Attachments.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "note":
+                        await Repositories.Notes.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "notebook":
+                        await Repositories.Notebooks.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "shortcut":
+                        await Repositories.Shortcuts.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "reminder":
+                        await Repositories.Reminders.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "relation":
+                        await Repositories.Relations.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "settings":
+                        await Repositories.Settings.UpsertAsync(userId, data, userId, dateSynced);
+                        break;
+                    case "vaultKey":
+                        userSettings.VaultKey = JsonSerializer.Deserialize<EncryptedData>(data);
+                        await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+                        break;
+                    default:
+                        throw new HubException("Invalid item type.");
                 }
+            });
 
-                return await unit.Commit() ? 1 : 0;
-            }
-            catch (Exception ex)
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-                throw ex;
-            }
+            return 1;
         }
 
         public async Task<bool> SyncCompleted(long dateSynced)
         {
             var userId = Context.User.FindFirstValue("sub");
-            try
-            {
-                UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
 
-                long lastSynced = Math.Max(dateSynced, userSettings.LastSynced);
+            UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
 
-                userSettings.LastSynced = lastSynced;
+            long lastSynced = dateSynced > userSettings.LastSynced ? dateSynced : userSettings.LastSynced;
 
-                await this.Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+            userSettings.LastSynced = lastSynced;
 
-                await Clients.OthersInGroup(userId).PushCompleted(lastSynced);
+            await this.Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
 
-                return true;
-            }
-            finally
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-            }
+            await Clients.OthersInGroup(userId).RemoteSyncCompleted(lastSynced);
+            return true;
         }
 
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, long, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, long lastSyncedTimestamp, int size, long maxBytes, int skipChunks)
+        public async IAsyncEnumerable<SyncTransferItem> FetchItems(long lastSyncedTimestamp, [EnumeratorCancellation]
+        CancellationToken cancellationToken)
         {
-            var chunksProcessed = 0;
-            for (int i = 0; i < collections.Length; i++)
+            var userId = Context.User.FindFirstValue("sub");
+
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+            if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
+                throw new HubException($"Provided timestamp value is too large. Server timestamp: {userSettings.LastSynced} Sent timestamp: {lastSyncedTimestamp}");
+
+            // var client = Clients.Caller;
+
+            if (lastSyncedTimestamp > 0 && userSettings.LastSynced == lastSyncedTimestamp)
             {
-                var type = types[i];
-
-                using var cursor = await collections[i](userId, lastSyncedTimestamp, size);
-
-                var chunk = new List<SyncItem>();
-                long totalBytes = 0;
-                long METADATA_BYTES = 5 * 1024;
-
-                while (await cursor.MoveNextAsync())
+                yield return new SyncTransferItem
                 {
-                    if (chunksProcessed++ < skipChunks) continue;
-                    foreach (var item in cursor.Current)
-                    {
-                        chunk.Add(item);
-                        totalBytes += item.Length + METADATA_BYTES;
-                        if (totalBytes >= maxBytes)
-                        {
-                            yield return new SyncTransferItemV2
-                            {
-                                Items = chunk,
-                                Type = type,
-                                Count = chunksProcessed
-                            };
+                    LastSynced = userSettings.LastSynced,
+                    Synced = true
+                };
+                yield break;
+            }
 
-                            totalBytes = 0;
-                            chunk.Clear();
-                        }
-                    }
-                }
-                if (chunk.Count > 0)
+
+            var attachments = await Repositories.Attachments.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var notes = await Repositories.Notes.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var notebooks = await Repositories.Notebooks.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var contents = await Repositories.Contents.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var settings = await Repositories.Settings.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var shortcuts = await Repositories.Shortcuts.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var reminders = await Repositories.Reminders.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var relations = await Repositories.Relations.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var collections = new Dictionary<string, IEnumerable<object>>
+            {
+                ["attachment"] = attachments,
+                ["note"] = notes,
+                ["notebook"] = notebooks,
+                ["content"] = contents,
+                ["shortcut"] = shortcuts,
+                ["reminder"] = reminders,
+                ["relation"] = relations,
+                ["settings"] = settings,
+            };
+
+            if (userSettings.VaultKey != null)
+            {
+                collections.Add("vaultKey", new object[] { userSettings.VaultKey });
+            }
+
+            var total = collections.Values.Sum((a) => a.Count());
+            if (total == 0)
+            {
+                yield return new SyncTransferItem
                 {
-                    if (chunksProcessed++ < skipChunks) continue;
-                    yield return new SyncTransferItemV2
+                    Synced = true,
+                    LastSynced = userSettings.LastSynced
+                };
+                yield break;
+            }
+
+            foreach (var collection in collections)
+            {
+                foreach (var item in collection.Value)
+                {
+                    if (item == null) continue;
+                    // Check the cancellation token regularly so that the server will stop producing items if the client disconnects.
+                    cancellationToken.ThrowIfCancellationRequested();
+                    yield return new SyncTransferItem
                     {
-                        Items = chunk,
-                        Type = type,
-                        Count = chunksProcessed
+                        LastSynced = userSettings.LastSynced,
+                        Synced = false,
+                        Item = JsonSerializer.Serialize(item),
+                        ItemType = collection.Key,
+                        Total = total,
                     };
                 }
             }
         }
 
-        public Task<SyncMetadata> RequestFetch(long lastSyncedTimestamp)
-        {
-            return RequestResumableFetch(lastSyncedTimestamp);
-        }
-
-        public async Task<SyncMetadata> RequestResumableFetch(long lastSyncedTimestamp, int cursor = 0)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-
-            if (GlobalSync.IsUserPushing(userId))
-            {
-                throw new HubException("Cannot fetch data while another sync is in progress. Please try again later.");
-            }
-
-            SyncEventCounterSource.Log.Fetch();
-
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
-            {
-                throw new HubException($"Provided timestamp value is too large. Server timestamp: {userSettings.LastSynced} Sent timestamp: {lastSyncedTimestamp}. Please run a Force Sync to fix this issue.");
-            }
-            // var client = Clients.Caller;
-
-            if (lastSyncedTimestamp > 0 && userSettings.LastSynced == lastSyncedTimestamp)
-            {
-                return new SyncMetadata
-                {
-                    LastSynced = userSettings.LastSynced,
-                };
-            }
-
-            var isResumable = lastSyncedTimestamp == 0;
-            if (!isResumable) cursor = 0;
-
-            var chunks = PrepareChunks(
-                collections: new[] {
-                    Repositories.LegacySettings.FindItemsSyncedAfter,
-                    Repositories.Attachments.FindItemsSyncedAfter,
-                    Repositories.Notes.FindItemsSyncedAfter,
-                    Repositories.Notebooks.FindItemsSyncedAfter,
-                    Repositories.Contents.FindItemsSyncedAfter,
-                    Repositories.Shortcuts.FindItemsSyncedAfter,
-                    Repositories.Reminders.FindItemsSyncedAfter,
-                    Repositories.Relations.FindItemsSyncedAfter,
-                },
-                types: CollectionKeys,
-                userId,
-                lastSyncedTimestamp,
-                size: 1000,
-                maxBytes: 7 * 1024 * 1024,
-                skipChunks: cursor
-            );
-
-            await foreach (var chunk in chunks)
-            {
-                _ = await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10));
-            }
-
-            return new SyncMetadata
-            {
-                VaultKey = userSettings.VaultKey,
-                LastSynced = userSettings.LastSynced,
-            };
-        }
     }
 
     [MessagePack.MessagePackObject]
@@ -412,6 +258,8 @@ namespace Notesnook.API.Hubs
 
         [MessagePack.Key("types")]
         public string[] Types { get; set; }
+        [MessagePack.Key("ids")]
+        public string[] Ids { get; set; }
 
         [MessagePack.Key("total")]
         public int Total { get; set; }
@@ -441,33 +289,4 @@ namespace Notesnook.API.Hubs
         [MessagePack.Key("current")]
         public int Current { get; set; }
     }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncTransferItemV2
-    {
-        [MessagePack.Key("items")]
-        [JsonPropertyName("items")]
-        public IEnumerable<SyncItem> Items { get; set; }
-
-        [MessagePack.Key("type")]
-        [JsonPropertyName("type")]
-        public string Type { get; set; }
-        [MessagePack.Key("count")]
-        [JsonPropertyName("count")]
-        public int Count { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncMetadata
-    {
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
-
-        [MessagePack.Key("lastSynced")]
-        [JsonPropertyName("lastSynced")]
-        public long LastSynced { get; set; }
-        // [MessagePack.Key("total")]
-        // public long TotalItems { get; set; }
-    }
 }

Notesnook.API/Hubs/SyncV2Hub.cs

@@ -1,324 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Security.Claims;
-using System.Text.Json.Serialization;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.SignalR;
-using MongoDB.Driver;
-using Notesnook.API.Authorization;
-using Notesnook.API.Interfaces;
-using Notesnook.API.Models;
-using Notesnook.API.Services;
-using Streetwriters.Data.Interfaces;
-
-namespace Notesnook.API.Hubs
-{
-    public interface ISyncV2HubClient
-    {
-        Task<bool> SendItems(SyncTransferItemV2 transferItem);
-        Task<bool> SendVaultKey(EncryptedData vaultKey);
-        Task<bool> SendMonographs(IEnumerable<Monograph> monographs);
-        Task PushCompleted();
-    }
-
-    [Authorize("Sync")]
-    public class SyncV2Hub : Hub<ISyncV2HubClient>
-    {
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = [
-            "settingitem",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "color",
-            "tag",
-            "vault",
-            "relation", // relations must sync at the end to prevent invalid state
-        ];
-
-        public SyncV2Hub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
-        {
-            Repositories = syncItemsRepositoryAccessor;
-            unit = unitOfWork;
-        }
-
-        public override async Task OnConnectedAsync()
-        {
-            var result = new SyncRequirement().IsAuthorized(Context.User, new PathString("/hubs/sync/v2"));
-            if (!result.Succeeded)
-            {
-                var reason = result.AuthorizationFailure.FailureReasons.FirstOrDefault();
-                throw new HubException(reason?.Message ?? "Unauthorized");
-            }
-            var id = Context.User.FindFirstValue("sub");
-            await Groups.AddToGroupAsync(Context.ConnectionId, id);
-            await base.OnConnectedAsync();
-        }
-
-        private Action<IEnumerable<SyncItem>, string, long> MapTypeToUpsertAction(string type)
-        {
-            return type switch
-            {
-                "settingitem" => Repositories.Settings.UpsertMany,
-                "attachment" => Repositories.Attachments.UpsertMany,
-                "note" => Repositories.Notes.UpsertMany,
-                "notebook" => Repositories.Notebooks.UpsertMany,
-                "content" => Repositories.Contents.UpsertMany,
-                "shortcut" => Repositories.Shortcuts.UpsertMany,
-                "reminder" => Repositories.Reminders.UpsertMany,
-                "relation" => Repositories.Relations.UpsertMany,
-                "color" => Repositories.Colors.UpsertMany,
-                "vault" => Repositories.Vaults.UpsertMany,
-                "tag" => Repositories.Tags.UpsertMany,
-                _ => null,
-            };
-        }
-
-        private Func<string, IEnumerable<string>, bool, int, Task<IAsyncCursor<SyncItem>>> MapTypeToFindItemsAction(string type)
-        {
-            return type switch
-            {
-                "settingitem" => Repositories.Settings.FindItemsById,
-                "attachment" => Repositories.Attachments.FindItemsById,
-                "note" => Repositories.Notes.FindItemsById,
-                "notebook" => Repositories.Notebooks.FindItemsById,
-                "content" => Repositories.Contents.FindItemsById,
-                "shortcut" => Repositories.Shortcuts.FindItemsById,
-                "reminder" => Repositories.Reminders.FindItemsById,
-                "relation" => Repositories.Relations.FindItemsById,
-                "color" => Repositories.Colors.FindItemsById,
-                "vault" => Repositories.Vaults.FindItemsById,
-                "tag" => Repositories.Tags.FindItemsById,
-                _ => null,
-            };
-        }
-
-        public async Task<int> PushItems(string deviceId, SyncTransferItemV2 pushItem)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
-
-            SyncEventCounterSource.Log.PushV2();
-
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-            try
-            {
-
-                var UpsertItems = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception($"Invalid item type: {pushItem.Type}.");
-                UpsertItems(pushItem.Items, userId, 1);
-
-                if (!await unit.Commit()) return 0;
-
-                new SyncDeviceService(new SyncDevice(userId, deviceId)).AddIdsToOtherDevices(pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
-                return 1;
-            }
-            finally
-            {
-                stopwatch.Stop();
-                SyncEventCounterSource.Log.RecordPushDuration(stopwatch.ElapsedMilliseconds);
-            }
-        }
-
-        public async Task<bool> PushCompleted()
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            await Clients.OthersInGroup(userId).PushCompleted();
-            return true;
-        }
-
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, string[], bool, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, string[] ids, int size, bool resetSync, long maxBytes)
-        {
-            var itemsProcessed = 0;
-            for (int i = 0; i < collections.Length; i++)
-            {
-                var type = types[i];
-
-                var filteredIds = ids.Where((id) => id.EndsWith($":{type}")).Select((id) => id.Split(":")[0]).ToArray();
-                if (!resetSync && filteredIds.Length == 0) continue;
-
-                using var cursor = await collections[i](userId, filteredIds, resetSync, size);
-
-                var chunk = new List<SyncItem>();
-                long totalBytes = 0;
-                long METADATA_BYTES = 5 * 1024;
-
-                while (await cursor.MoveNextAsync())
-                {
-                    foreach (var item in cursor.Current)
-                    {
-                        chunk.Add(item);
-                        totalBytes += item.Length + METADATA_BYTES;
-                        if (totalBytes >= maxBytes)
-                        {
-                            itemsProcessed += chunk.Count;
-                            yield return new SyncTransferItemV2
-                            {
-                                Items = chunk,
-                                Type = type,
-                                Count = itemsProcessed
-                            };
-
-                            totalBytes = 0;
-                            chunk.Clear();
-                        }
-                    }
-                }
-                if (chunk.Count > 0)
-                {
-                    itemsProcessed += chunk.Count;
-                    yield return new SyncTransferItemV2
-                    {
-                        Items = chunk,
-                        Type = type,
-                        Count = itemsProcessed
-                    };
-                }
-            }
-        }
-
-        public async Task<SyncV2Metadata> RequestFetch(string deviceId)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
-
-            SyncEventCounterSource.Log.FetchV2();
-
-            var device = new SyncDevice(userId, deviceId);
-            var deviceService = new SyncDeviceService(device);
-            if (!deviceService.IsDeviceRegistered()) deviceService.RegisterDevice();
-
-            device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-
-            var isResetSync = deviceService.IsSyncReset();
-            if (!deviceService.IsUnsynced() &&
-                !deviceService.IsSyncPending() &&
-                !isResetSync)
-                return new SyncV2Metadata { Synced = true };
-
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-            try
-            {
-                string[] ids = deviceService.FetchUnsyncedIds();
-
-                var chunks = PrepareChunks(
-                    collections: [
-                        Repositories.Settings.FindItemsById,
-                    Repositories.Attachments.FindItemsById,
-                    Repositories.Notes.FindItemsById,
-                    Repositories.Notebooks.FindItemsById,
-                    Repositories.Contents.FindItemsById,
-                    Repositories.Shortcuts.FindItemsById,
-                    Repositories.Reminders.FindItemsById,
-                    Repositories.Colors.FindItemsById,
-                    Repositories.Tags.FindItemsById,
-                    Repositories.Vaults.FindItemsById,
-                    Repositories.Relations.FindItemsById,
-                    ],
-                    types: CollectionKeys,
-                    userId,
-                    ids,
-                    size: 1000,
-                    resetSync: isResetSync,
-                    maxBytes: 7 * 1024 * 1024
-                );
-
-                var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId.Equals(userId));
-                if (userSettings.VaultKey != null)
-                {
-                    if (!await Clients.Caller.SendVaultKey(userSettings.VaultKey).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected vault key.");
-                }
-
-
-                await foreach (var chunk in chunks)
-                {
-                    if (!await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected sent items.");
-
-                    if (!isResetSync)
-                    {
-                        var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
-                        ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
-                        deviceService.WritePendingIds(ids);
-                    }
-                }
-
-                var unsyncedMonographs = ids.Where((id) => id.EndsWith(":monograph")).ToHashSet();
-                var unsyncedMonographIds = unsyncedMonographs.Select((id) => id.Split(":")[0]).ToArray();
-                var userMonographs = isResetSync
-                    ? await Repositories.Monographs.FindAsync(m => m.UserId == userId)
-                    : await Repositories.Monographs.FindAsync(m => m.UserId == userId && unsyncedMonographIds.Contains(m.ItemId));
-
-                if (userMonographs.Any() && !await Clients.Caller.SendMonographs(userMonographs).WaitAsync(TimeSpan.FromMinutes(10)))
-                    throw new HubException("Client rejected monographs.");
-
-                deviceService.Reset();
-
-                return new SyncV2Metadata
-                {
-                    Synced = true,
-                };
-            }
-            finally
-            {
-                stopwatch.Stop();
-                SyncEventCounterSource.Log.RecordFetchDuration(stopwatch.ElapsedMilliseconds);
-            }
-        }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncV2Metadata
-    {
-        [MessagePack.Key("synced")]
-        [JsonPropertyName("synced")]
-        public bool Synced { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncV2TransferItem
-    {
-        [MessagePack.Key("items")]
-        [JsonPropertyName("items")]
-        public IEnumerable<SyncItem> Items { get; set; }
-
-        [MessagePack.Key("type")]
-        [JsonPropertyName("type")]
-        public string Type { get; set; }
-
-        [MessagePack.Key("final")]
-        [JsonPropertyName("final")]
-        public bool Final { get; set; }
-
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
-    }
-}

Notesnook.API/Interfaces/IS3Service.cs

@@ -30,7 +30,7 @@ namespace Notesnook.API.Interfaces
     {
         Task DeleteObjectAsync(string userId, string name);
         Task DeleteDirectoryAsync(string userId);
-        Task<long> GetObjectSizeAsync(string userId, string name);
+        Task<long?> GetObjectSizeAsync(string userId, string name);
         string GetUploadObjectUrl(string userId, string name);
         string GetDownloadObjectUrl(string userId, string name);
         Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string uploadId = null);

Notesnook.API/Interfaces/ISyncItem.cs

@@ -0,0 +1,43 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Text.Json.Serialization;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Bson.Serialization.Serializers;
+using Notesnook.API.Models;
+using Streetwriters.Common.Attributes;
+using Streetwriters.Common.Converters;
+using Streetwriters.Common.Interfaces;
+
+namespace Notesnook.API.Interfaces
+{
+    [BsonSerializer(typeof(ImpliedImplementationInterfaceSerializer<ISyncItem, SyncItem>))]
+    [JsonInterfaceConverter(typeof(InterfaceConverter<ISyncItem, SyncItem>))]
+    public interface ISyncItem
+    {
+        long DateSynced
+        {
+            get; set;
+        }
+
+        string UserId { get; set; }
+        string Algorithm { get; set; }
+        string IV { get; set; }
+    }
+}

Notesnook.API/Interfaces/ISyncItemsRepositoryAccessor.cs

@@ -26,18 +26,14 @@ namespace Notesnook.API.Interfaces
 {
     public interface ISyncItemsRepositoryAccessor
     {
-        SyncItemsRepository Notes { get; }
-        SyncItemsRepository Notebooks { get; }
-        SyncItemsRepository Shortcuts { get; }
-        SyncItemsRepository Reminders { get; }
-        SyncItemsRepository Relations { get; }
-        SyncItemsRepository Contents { get; }
-        SyncItemsRepository LegacySettings { get; }
-        SyncItemsRepository Attachments { get; }
-        SyncItemsRepository Settings { get; }
-        SyncItemsRepository Colors { get; }
-        SyncItemsRepository Vaults { get; }
-        SyncItemsRepository Tags { get; }
+        SyncItemsRepository<Note> Notes { get; }
+        SyncItemsRepository<Notebook> Notebooks { get; }
+        SyncItemsRepository<Shortcut> Shortcuts { get; }
+        SyncItemsRepository<Reminder> Reminders { get; }
+        SyncItemsRepository<Relation> Relations { get; }
+        SyncItemsRepository<Content> Contents { get; }
+        SyncItemsRepository<Setting> Settings { get; }
+        SyncItemsRepository<Attachment> Attachments { get; }
         Repository<UserSettings> UsersSettings { get; }
         Repository<Monograph> Monographs { get; }
     }

Notesnook.API/Interfaces/IUserService.cs

@@ -17,19 +17,19 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.Threading;
 using System.Threading.Tasks;
-using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
+using Streetwriters.Common.Interfaces;
 
 namespace Notesnook.API.Interfaces
 {
     public interface IUserService
     {
         Task CreateUserAsync();
-        Task DeleteUserAsync(string userId);
-        Task DeleteUserAsync(string userId, string jti, string password);
+        Task<bool> DeleteUserAsync(string userId, string jti);
         Task<bool> ResetUserAsync(string userId, bool removeAttachments);
-        Task<UserResponse> GetUserAsync(string userId);
-        Task SetUserKeysAsync(string userId, UserKeys keys);
+        Task<UserResponse> GetUserAsync(bool repair = true);
+        Task SetUserAttachmentsKeyAsync(string userId, IEncrypted key);
     }
 }

Notesnook.API/Models/Algorithms.cs

@@ -17,10 +17,17 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.Collections.Generic;
+
 namespace Notesnook.API.Models
 {
     public class Algorithms
     {
-        public static string Default => "xcha-argon2i13-7";
+        public const string Default = "xcha-argon2i13-7";
+        static readonly List<string> ALGORITHMS = new List<string> { Algorithms.Default };
+        public static bool IsValidAlgorithm(string algorithm)
+        {
+            return ALGORITHMS.Contains(algorithm);
+        }
     }
 }

Notesnook.API/Models/Announcement.cs

@@ -22,9 +22,11 @@ using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
+    [BsonCollection("notesnook", "announcements")]
     public class Announcement
     {
         public Announcement()

Notesnook.API/Models/CompleteMultipartUploadRequestWrapper.cs

@@ -1,30 +0,0 @@
-using System.Collections.Generic;
-using Amazon.S3.Model;
-
-namespace Notesnook.API.Models;
-
-public class CompleteMultipartUploadRequestWrapper
-{
-    public string Key { get; set; }
-    public List<PartETagWrapper> PartETags { get; set; }
-    public string UploadId { get; set; }
-
-    public CompleteMultipartUploadRequest ToRequest()
-    {
-        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest();
-        completeMultipartUploadRequest.Key = Key;
-        completeMultipartUploadRequest.UploadId = UploadId;
-        completeMultipartUploadRequest.PartETags = [];
-        foreach (var partETagWrapper in PartETags)
-        {
-            var partETag = new PartETag
-            {
-                PartNumber = partETagWrapper.PartNumber,
-                ETag = partETagWrapper.ETag
-            };
-            completeMultipartUploadRequest.PartETags.Add(partETag);
-        }
-
-        return completeMultipartUploadRequest;
-    }
-}

Notesnook.API/Models/DeleteAccountForm.cs

@@ -1,13 +0,0 @@
-using System.ComponentModel.DataAnnotations;
-
-namespace Notesnook.API.Models
-{
-    public class DeleteAccountForm
-    {
-        [Required]
-        public string Password
-        {
-            get; set;
-        }
-    }
-}

Notesnook.API/Models/EncryptedData.cs

@@ -25,10 +25,8 @@ using System.Text.Json.Serialization;
 
 namespace Notesnook.API.Models
 {
-    [MessagePack.MessagePackObject]
     public class EncryptedData : IEncrypted
     {
-        [MessagePack.Key("iv")]
         [JsonPropertyName("iv")]
         [BsonElement("iv")]
         [DataMember(Name = "iv")]
@@ -37,7 +35,6 @@ namespace Notesnook.API.Models
             get; set;
         }
 
-        [MessagePack.Key("cipher")]
         [JsonPropertyName("cipher")]
         [BsonElement("cipher")]
         [DataMember(Name = "cipher")]
@@ -46,30 +43,14 @@ namespace Notesnook.API.Models
             get; set;
         }
 
-        [MessagePack.Key("length")]
         [JsonPropertyName("length")]
         [BsonElement("length")]
         [DataMember(Name = "length")]
         public long Length { get; set; }
 
-        [MessagePack.Key("salt")]
         [JsonPropertyName("salt")]
         [BsonElement("salt")]
         [DataMember(Name = "salt")]
         public string Salt { get; set; }
-
-        public override bool Equals(object obj)
-        {
-            if (obj is EncryptedData encryptedData)
-            {
-                return IV == encryptedData.IV && Salt == encryptedData.Salt && Cipher == encryptedData.Cipher && Length == encryptedData.Length;
-            }
-            return base.Equals(obj);
-        }
-
-        public bool IsEmpty()
-        {
-            return this.Cipher == null && this.IV == null && this.Length == 0 && this.Salt == null;
-        }
     }
 }

Notesnook.API/Models/Monograph.cs

@@ -20,50 +20,22 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using System.Text.Json.Serialization;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
-using System.Runtime.Serialization;
+using Notesnook.API.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
-    public class ObjectWithId
-    {
-        [BsonId]
-        [BsonIgnoreIfDefault]
-        [BsonRepresentation(BsonType.ObjectId)]
-        public string Id
-        {
-            get; set;
-        }
-
-        public string ItemId
-        {
-            get; set;
-        }
-    }
-
-    public class Monograph
+    [BsonCollection("notesnook", "monographs")]
+    public class Monograph : IMonograph
     {
         public Monograph()
         {
             Id = ObjectId.GenerateNewId().ToString();
         }
 
-        [DataMember(Name = "id")]
-        [JsonPropertyName("id")]
-        [MessagePack.Key("id")]
-        public string ItemId
-        {
-            get; set;
-        }
-
         [BsonId]
-        [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
-        [JsonIgnore]
-        [MessagePack.IgnoreMember]
-        public string Id
-        {
-            get; set;
-        }
+        public string Id { get; set; }
 
         [JsonPropertyName("title")]
         public string Title { get; set; }
@@ -86,11 +58,5 @@ namespace Notesnook.API.Models
 
         [JsonIgnore]
         public byte[] CompressedContent { get; set; }
-
-        [JsonPropertyName("password")]
-        public EncryptedData Password { get; set; }
-
-        [JsonPropertyName("deleted")]
-        public bool Deleted { get; set; }
     }
 }

Notesnook.API/Models/PartETagWrapper.cs

@@ -1,7 +0,0 @@
-namespace Notesnook.API.Models;
-
-public class PartETagWrapper
-{
-    public int PartNumber { get; set; }
-    public string ETag { get; set; }
-}

Notesnook.API/Models/Responses/UserResponse.cs

@@ -12,15 +12,9 @@ namespace Notesnook.API.Models.Responses
         [JsonPropertyName("attachmentsKey")]
         public EncryptedData AttachmentsKey { get; set; }
 
-        [JsonPropertyName("monographPasswordsKey")]
-        public EncryptedData MonographPasswordsKey { get; set; }
-
         [JsonPropertyName("subscription")]
         public ISubscription Subscription { get; set; }
 
-        [JsonPropertyName("profile")]
-        public EncryptedData Profile { get; set; }
-
         [JsonIgnore]
         public bool Success { get; set; }
         public int StatusCode { get; set; }

Notesnook.API/Models/SyncItem.cs

@@ -17,24 +17,19 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System;
 using System.ComponentModel.DataAnnotations;
 using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
 using MongoDB.Bson;
-using MongoDB.Bson.IO;
-using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Attributes;
-using MongoDB.Bson.Serialization.Serializers;
 using Notesnook.API.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
-    [MessagePack.MessagePackObject]
-    public class SyncItem
+    public class SyncItem : ISyncItem
     {
         [IgnoreDataMember]
-        [MessagePack.IgnoreMember]
         [JsonPropertyName("dateSynced")]
         public long DateSynced
         {
@@ -43,7 +38,6 @@ namespace Notesnook.API.Models
 
         [DataMember(Name = "userId")]
         [JsonPropertyName("userId")]
-        [MessagePack.Key("userId")]
         public string UserId
         {
             get; set;
@@ -51,7 +45,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("iv")]
         [DataMember(Name = "iv")]
-        [MessagePack.Key("iv")]
         [Required]
         public string IV
         {
@@ -61,7 +54,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("cipher")]
         [DataMember(Name = "cipher")]
-        [MessagePack.Key("cipher")]
         [Required]
         public string Cipher
         {
@@ -70,7 +62,6 @@ namespace Notesnook.API.Models
 
         [DataMember(Name = "id")]
         [JsonPropertyName("id")]
-        [MessagePack.Key("id")]
         public string ItemId
         {
             get; set;
@@ -80,7 +71,6 @@ namespace Notesnook.API.Models
         [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
-        [MessagePack.IgnoreMember]
         public ObjectId Id
         {
             get; set;
@@ -88,7 +78,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("length")]
         [DataMember(Name = "length")]
-        [MessagePack.Key("length")]
         [Required]
         public long Length
         {
@@ -97,7 +86,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("v")]
         [DataMember(Name = "v")]
-        [MessagePack.Key("v")]
         [Required]
         public double Version
         {
@@ -106,7 +94,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("alg")]
         [DataMember(Name = "alg")]
-        [MessagePack.Key("alg")]
         [Required]
         public string Algorithm
         {
@@ -114,92 +101,27 @@ namespace Notesnook.API.Models
         } = Algorithms.Default;
     }
 
-    public class SyncItemBsonSerializer : SerializerBase<SyncItem>
-    {
-        public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, SyncItem value)
-        {
-            var writer = context.Writer;
-            writer.WriteStartDocument();
+    [BsonCollection("notesnook", "attachments")]
+    public class Attachment : SyncItem { }
 
-            if (value.Id != ObjectId.Empty)
-            {
-                writer.WriteName("_id");
-                writer.WriteObjectId(value.Id);
-            }
+    [BsonCollection("notesnook", "content")]
+    public class Content : SyncItem { }
 
-            writer.WriteName("DateSynced");
-            writer.WriteInt64(value.DateSynced);
+    [BsonCollection("notesnook", "notes")]
+    public class Note : SyncItem { }
 
-            writer.WriteName("UserId");
-            writer.WriteString(value.UserId);
+    [BsonCollection("notesnook", "notebooks")]
+    public class Notebook : SyncItem { }
 
-            writer.WriteName("IV");
-            writer.WriteString(value.IV);
+    [BsonCollection("notesnook", "relations")]
+    public class Relation : SyncItem { }
 
-            writer.WriteName("Cipher");
-            writer.WriteString(value.Cipher);
+    [BsonCollection("notesnook", "reminders")]
+    public class Reminder : SyncItem { }
 
-            writer.WriteName("ItemId");
-            writer.WriteString(value.ItemId);
+    [BsonCollection("notesnook", "settings")]
+    public class Setting : SyncItem { }
 
-            writer.WriteName("Length");
-            writer.WriteInt64(value.Length);
-
-            writer.WriteName("Version");
-            writer.WriteDouble(value.Version);
-
-            writer.WriteName("Algorithm");
-            writer.WriteString(value.Algorithm);
-
-            writer.WriteEndDocument();
-        }
-
-        public override SyncItem Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
-        {
-            var syncItem = new SyncItem();
-            var bsonReader = context.Reader;
-            bsonReader.ReadStartDocument();
-
-            while (bsonReader.ReadBsonType() != BsonType.EndOfDocument)
-            {
-                var fieldName = bsonReader.ReadName();
-
-                switch (fieldName)
-                {
-                    case "DateSynced":
-                        syncItem.DateSynced = bsonReader.ReadInt64();
-                        break;
-                    case "UserId":
-                        syncItem.UserId = bsonReader.ReadString();
-                        break;
-                    case "IV":
-                        syncItem.IV = bsonReader.ReadString();
-                        break;
-                    case "Cipher":
-                        syncItem.Cipher = bsonReader.ReadString();
-                        break;
-                    case "ItemId":
-                        syncItem.ItemId = bsonReader.ReadString();
-                        break;
-                    case "_id":
-                        syncItem.Id = bsonReader.ReadObjectId();
-                        break;
-                    case "Length":
-                        syncItem.Length = bsonReader.ReadInt64();
-                        break;
-                    case "Version":
-                        syncItem.Version = bsonReader.ReadDouble();
-                        break;
-                    case "Algorithm":
-                        syncItem.Algorithm = bsonReader.ReadString();
-                        break;
-                    default:
-                        bsonReader.SkipValue();
-                        break;
-                }
-            }
-            bsonReader.ReadEndDocument();
-            return syncItem;
-        }
-    }
+    [BsonCollection("notesnook", "shortcuts")]
+    public class Shortcut : SyncItem { }
 }

Notesnook.API/Models/UserSettings.cs

@@ -20,9 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Notesnook.API.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
+    [BsonCollection("notesnook", "user_settings")]
     public class UserSettings : IUserSettings
     {
         public UserSettings()
@@ -34,7 +36,6 @@ namespace Notesnook.API.Models
         public string Salt { get; set; }
         public EncryptedData VaultKey { get; set; }
         public EncryptedData AttachmentsKey { get; set; }
-        public EncryptedData MonographPasswordsKey { get; set; }
 
         [BsonId]
         [BsonRepresentation(BsonType.ObjectId)]

Notesnook.API/Notesnook.API.csproj

@@ -1,23 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <StartupObject>Notesnook.API.Program</StartupObject>
+    <LangVersion>10.0</LangVersion>
+    <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
+    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AWSSDK.Core" Version="3.7.304.31" />
+    <PackageReference Include="AWSSDK.Core" Version="3.7.12.5" />
     <PackageReference Include="DotNetEnv" Version="2.3.0" />
-    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="6.2.0" />
+    <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="3.0.1" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="6.0.0" />
     <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="6.0.1-rc2.2" />
-    <PackageReference Include="AWSSDK.S3" Version="3.7.310.8" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.9.21" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="6.0.3" />
     <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Https" Version="2.2.0" />
-    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-alpha.2" />
-    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
-  </ItemGroup>
+ </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Streetwriters.Common\Streetwriters.Common.csproj" />

Notesnook.API/Program.cs

@@ -17,13 +17,15 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+#if !DEBUG
+using System.Net;
+#endif
 using System;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
 using Streetwriters.Common;
-using System.Net;
+using System.Linq;
 
 namespace Notesnook.API
 {
@@ -31,7 +33,7 @@ namespace Notesnook.API
     {
         public static async Task Main(string[] args)
         {
-#if (DEBUG || STAGING)
+#if DEBUG
             DotNetEnv.Env.TraversePath().Load(".env.local");
 #else
             DotNetEnv.Env.TraversePath().Load(".env");
@@ -57,7 +59,6 @@ namespace Notesnook.API
                                 listenerOptions.UseHttps(Servers.NotesnookAPI.SSLCertificate);
                             });
                         }
-                        options.Listen(IPAddress.Parse("127.0.0.1"), 5067);
                     });
                 });
     }

Notesnook.API/Repositories/SyncItemsRepository.cs

@@ -19,180 +19,82 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
-using IdentityModel;
-using Microsoft.VisualBasic;
-using MongoDB.Bson;
 using MongoDB.Driver;
-using Notesnook.API.Hubs;
-using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
-using Streetwriters.Common;
-using Streetwriters.Data.DbContexts;
-using Streetwriters.Data.Interfaces;
-using Streetwriters.Data.Repositories;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Repositories
 {
-    public class SyncItemsRepository : Repository<SyncItem>
+    public class SyncItemsRepository<T> where T : SyncItem
     {
-        private readonly string collectionName;
-        public SyncItemsRepository(IDbContext dbContext, IMongoCollection<SyncItem> collection) : base(dbContext, collection)
+        const string BASE_DATA_DIR = "data";
+        private string GetCollectionName()
         {
-            this.collectionName = collection.CollectionNamespace.CollectionName;
+            var attribute = (BsonCollectionAttribute)typeof(T).GetCustomAttributes(
+                    typeof(BsonCollectionAttribute),
+                    true).FirstOrDefault();
+            if (string.IsNullOrEmpty(attribute.CollectionName) || string.IsNullOrEmpty(attribute.DatabaseName)) throw new Exception("Could not get a valid collection or database name.");
+            return attribute.CollectionName;
         }
 
-        private readonly List<string> ALGORITHMS = [Algorithms.Default];
-        private bool IsValidAlgorithm(string algorithm)
+        private string GetUserDirectoryPath(string userId)
         {
-            return ALGORITHMS.Contains(algorithm);
+            return System.IO.Path.Join(BASE_DATA_DIR, userId, GetCollectionName());
         }
 
-        public Task<long> CountItemsSyncedAfterAsync(string userId, long timestamp)
+        private IEnumerable<string> EnumerateItems(string userId, string searchPattern = "*")
         {
-            var filter = Builders<SyncItem>.Filter.And(Builders<SyncItem>.Filter.Gt("DateSynced", timestamp), Builders<SyncItem>.Filter.Eq("UserId", userId));
-            return Collection.CountDocumentsAsync(filter);
-        }
-        public Task<IAsyncCursor<SyncItem>> FindItemsSyncedAfter(string userId, long timestamp, int batchSize)
-        {
-            var filter = Builders<SyncItem>.Filter.And(Builders<SyncItem>.Filter.Gt("DateSynced", timestamp), Builders<SyncItem>.Filter.Eq("UserId", userId));
-            return Collection.FindAsync(filter, new FindOptions<SyncItem>
+            try
             {
-                BatchSize = batchSize,
-                AllowDiskUse = true,
-                AllowPartialResults = false,
-                NoCursorTimeout = true,
-                Sort = new SortDefinitionBuilder<SyncItem>().Ascending("_id")
-            });
+                return System.IO.Directory.EnumerateFiles(GetUserDirectoryPath(userId), searchPattern, System.IO.SearchOption.TopDirectoryOnly);
+            }
+            catch
+            {
+                return new string[] { };
+            }
         }
 
-        public Task<IAsyncCursor<SyncItem>> FindItemsById(string userId, IEnumerable<string> ids, bool all, int batchSize)
+        private string FindItemById(string userId, string id)
         {
-            var filters = new List<FilterDefinition<SyncItem>>(new[] { Builders<SyncItem>.Filter.Eq("UserId", userId) });
-
-            if (!all) filters.Add(Builders<SyncItem>.Filter.In("ItemId", ids));
-
-            return Collection.FindAsync(Builders<SyncItem>.Filter.And(filters), new FindOptions<SyncItem>
+            try
             {
-                BatchSize = batchSize,
-                AllowDiskUse = true,
-                AllowPartialResults = false,
-                NoCursorTimeout = true
+                var files = Directory.GetFiles(GetUserDirectoryPath(userId), $"{id}-*", System.IO.SearchOption.TopDirectoryOnly);
+                return files.Length > 0 ? files[0] : null;
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public async Task<IEnumerable<string>> GetItemsSyncedAfterAsync(string userId, long timestamp)
+        {
+            var items = new List<string>();
+            await Parallel.ForEachAsync(EnumerateItems(userId), async (file, ct) =>
+            {
+                var parts = file.Split("-");
+                var id = parts[0];
+                var dateSynced = long.Parse(parts[1]);
+                if (dateSynced > timestamp) items.Add(await File.ReadAllTextAsync(file));
             });
+            return items;
         }
 
         public void DeleteByUserId(string userId)
         {
-            var filter = Builders<SyncItem>.Filter.Eq("UserId", userId);
-            var writes = new List<WriteModel<SyncItem>>
-            {
-                new DeleteManyModel<SyncItem>(filter)
-            };
-            dbContext.AddCommand((handle, ct) => Collection.BulkWriteAsync(handle, writes, options: null, ct));
+            Directory.Delete(GetUserDirectoryPath(userId), true);
         }
 
-        public void Upsert(SyncItem item, string userId, long dateSynced)
+        public async Task UpsertAsync(string id, string item, string userId, long dateSynced)
         {
-            if (item.Length > 15 * 1024 * 1024)
-            {
-                throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
-            }
-
-            if (!IsValidAlgorithm(item.Algorithm))
-            {
-                throw new Exception($"Invalid alg identifier {item.Algorithm}");
-            }
-
-            // Handle case where the cipher is corrupted.
-            if (!IsBase64String(item.Cipher))
-            {
-                Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
-                throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
-            }
-
-            item.DateSynced = dateSynced;
-            item.UserId = userId;
-
-            var filter = Builders<SyncItem>.Filter.And(
-                Builders<SyncItem>.Filter.Eq("UserId", userId),
-                Builders<SyncItem>.Filter.Eq("ItemId", item.ItemId)
-            );
-
-            dbContext.AddCommand((handle, ct) => Collection.ReplaceOneAsync(handle, filter, item, new ReplaceOptions { IsUpsert = true }, ct));
-        }
-
-        public void UpsertMany(IEnumerable<SyncItem> items, string userId, long dateSynced)
-        {
-            var userIdFilter = Builders<SyncItem>.Filter.Eq("UserId", userId);
-            var writes = new List<WriteModel<SyncItem>>();
-            foreach (var item in items)
-            {
-                if (item.Length > 15 * 1024 * 1024)
-                {
-                    throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
-                }
-
-                if (!IsValidAlgorithm(item.Algorithm))
-                {
-                    throw new Exception($"Invalid alg identifier {item.Algorithm}");
-                }
-
-                // Handle case where the cipher is corrupted.
-                if (!IsBase64String(item.Cipher))
-                {
-                    Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
-                    throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
-                }
-
-                var filter = Builders<SyncItem>.Filter.And(
-                    userIdFilter,
-                    Builders<SyncItem>.Filter.Eq("ItemId", item.ItemId)
-                );
-
-                item.DateSynced = dateSynced;
-                item.UserId = userId;
-
-                writes.Add(new ReplaceOneModel<SyncItem>(filter, item)
-                {
-                    IsUpsert = true
-                });
-            }
-            dbContext.AddCommand((handle, ct) => Collection.BulkWriteAsync(handle, writes, options: new BulkWriteOptions { IsOrdered = false }, ct));
-        }
-
-        private static bool IsBase64String(string value)
-        {
-            if (value == null || value.Length == 0 || value.Contains(' ') || value.Contains('\t') || value.Contains('\r') || value.Contains('\n'))
-                return false;
-            var index = value.Length - 1;
-            if (value[index] == '=')
-                index--;
-            if (value[index] == '=')
-                index--;
-            for (var i = 0; i <= index; i++)
-                if (IsInvalidBase64Char(value[i]))
-                    return false;
-            return true;
-        }
-
-        private static bool IsInvalidBase64Char(char value)
-        {
-            var code = (int)value;
-            // 1 - 9
-            if (code >= 48 && code <= 57)
-                return false;
-            // A - Z
-            if (code >= 65 && code <= 90)
-                return false;
-            // a - z
-            if (code >= 97 && code <= 122)
-                return false;
-            // - & _
-            return code != 45 && code != 95;
+            Directory.CreateDirectory(GetUserDirectoryPath(userId));
+            var oldPath = FindItemById(userId, id);
+            var newPath = Path.Join(GetUserDirectoryPath(userId), $"{id}-{dateSynced}");
+            await File.WriteAllTextAsync(newPath, item);
+            if (oldPath != null) File.Delete(oldPath);
         }
     }
 }

Notesnook.API/Services/S3Service.cs

@@ -42,8 +42,7 @@ namespace Notesnook.API.Services
 
     public class S3Service : IS3Service
     {
-        private readonly string BUCKET_NAME = Constants.S3_BUCKET_NAME ?? "";
-        private readonly string INTERNAL_BUCKET_NAME = Constants.S3_INTERNAL_BUCKET_NAME ?? "";
+        private readonly string BUCKET_NAME = "nn-attachments";
         private AmazonS3Client S3Client { get; }
 
         // When running in a dockerized environment the sync server doesn't have access
@@ -61,7 +60,7 @@ namespace Notesnook.API.Services
         {
             var config = new AmazonS3Config
             {
-#if (DEBUG || STAGING)
+#if DEBUG
                 ServiceURL = Servers.S3Server.ToString(),
 #else
                 ServiceURL = Constants.S3_SERVICE_URL,
@@ -71,7 +70,7 @@ namespace Notesnook.API.Services
                 SignatureMethod = SigningAlgorithm.HmacSHA256,
                 SignatureVersion = "4"
             };
-#if (DEBUG || STAGING)
+#if DEBUG
             S3Client = new AmazonS3Client("S3RVER", "S3RVER", config);
 #else
             S3Client = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, config);
@@ -97,7 +96,7 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (objectName == null) throw new Exception("Invalid object name."); ;
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(BUCKET_NAME, objectName);
 
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode)))
                 throw new Exception("Could not delete object.");
@@ -107,7 +106,7 @@ namespace Notesnook.API.Services
         {
             var request = new ListObjectsV2Request
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = BUCKET_NAME,
                 Prefix = userId,
             };
 
@@ -127,10 +126,10 @@ namespace Notesnook.API.Services
 
             if (keys.Count <= 0) return;
 
-            var deleteObjectsResponse = await GetS3Client(S3ClientMode.INTERNAL)
+            var deleteObjectsResponse = await S3Client
             .DeleteObjectsAsync(new DeleteObjectsRequest
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = BUCKET_NAME,
                 Objects = keys,
             });
 
@@ -138,20 +137,14 @@ namespace Notesnook.API.Services
                 throw new Exception("Could not delete directory.");
         }
 
-        public async Task<long> GetObjectSizeAsync(string userId, string name)
+        public async Task<long?> GetObjectSizeAsync(string userId, string name)
         {
             var url = this.GetPresignedURL(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
-            if (url == null) return 0;
+            if (url == null) return null;
 
             var request = new HttpRequestMessage(HttpMethod.Head, url);
             var response = await httpClient.SendAsync(request);
-            const long MAX_SIZE = 513 * 1024 * 1024; // 512 MB
-            if (!Constants.IS_SELF_HOSTED && response.Content.Headers.ContentLength >= MAX_SIZE)
-            {
-                await this.DeleteObjectAsync(userId, name);
-                throw new Exception("File size exceeds the maximum allowed size.");
-            }
-            return response.Content.Headers.ContentLength ?? 0;
+            return response.Content.Headers.ContentLength;
         }
 
 
@@ -176,7 +169,7 @@ namespace Notesnook.API.Services
 
             if (string.IsNullOrEmpty(uploadId))
             {
-                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
+                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(BUCKET_NAME, objectName);
                 if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to initiate multipart upload.");
 
                 uploadId = response.UploadId;
@@ -200,7 +193,7 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName, uploadId);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(BUCKET_NAME, objectName, uploadId);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to abort multipart upload.");
         }
 
@@ -210,7 +203,7 @@ namespace Notesnook.API.Services
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
             uploadRequest.Key = objectName;
-            uploadRequest.BucketName = GetBucketName(S3ClientMode.INTERNAL);
+            uploadRequest.BucketName = BUCKET_NAME;
             var response = await GetS3Client(S3ClientMode.INTERNAL).CompleteMultipartUploadAsync(uploadRequest);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to complete multipart upload.");
         }
@@ -220,38 +213,36 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) return null;
 
-            var client = GetS3Client(mode);
             var request = new GetPreSignedUrlRequest
             {
-                BucketName = GetBucketName(mode),
+                BucketName = BUCKET_NAME,
                 Expires = System.DateTime.Now.AddHours(1),
                 Verb = httpVerb,
                 Key = objectName,
-#if (DEBUG || STAGING)
+#if DEBUG
                 Protocol = Protocol.HTTP,
 #else
-                Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
+                Protocol = Constants.IS_SELF_HOSTED ? Protocol.HTTP : Protocol.HTTPS,
 #endif
             };
-            return client.GetPreSignedURL(request);
+            return GetS3Client(mode).GetPreSignedURL(request);
         }
 
-        private string GetPresignedURLForUploadPart(string objectName, string uploadId, int partNumber, S3ClientMode mode = S3ClientMode.EXTERNAL)
+        private string GetPresignedURLForUploadPart(string objectName, string uploadId, int partNumber)
         {
 
-            var client = GetS3Client(mode);
-            return client.GetPreSignedURL(new GetPreSignedUrlRequest
+            return GetS3Client().GetPreSignedURL(new GetPreSignedUrlRequest
             {
-                BucketName = GetBucketName(mode),
+                BucketName = BUCKET_NAME,
                 Expires = System.DateTime.Now.AddHours(1),
                 Verb = HttpVerb.PUT,
                 Key = objectName,
                 PartNumber = partNumber,
                 UploadId = uploadId,
-#if (DEBUG || STAGING)
+#if DEBUG
                 Protocol = Protocol.HTTP,
 #else
-                Protocol = client.Config.ServiceURL.StartsWith("http://") ? Protocol.HTTP : Protocol.HTTPS,
+                Protocol = Constants.IS_SELF_HOSTED ? Protocol.HTTP : Protocol.HTTPS,
 #endif
             });
         }
@@ -272,11 +263,5 @@ namespace Notesnook.API.Services
             if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return S3InternalClient;
             return S3Client;
         }
-
-        string GetBucketName(S3ClientMode mode = S3ClientMode.EXTERNAL)
-        {
-            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return INTERNAL_BUCKET_NAME;
-            return BUCKET_NAME;
-        }
     }
 }

Notesnook.API/Services/SyncDeviceService.cs

@@ -1,233 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Notesnook.API.Services
-{
-    public struct SyncDevice(string userId, string deviceId)
-    {
-        public readonly string DeviceId => deviceId;
-        public readonly string UserId => userId;
-
-        public string UserSyncDirectoryPath = CreateFilePath(userId);
-        public string UserDeviceDirectoryPath = CreateFilePath(userId, deviceId);
-        public string PendingIdsFilePath = CreateFilePath(userId, deviceId, "pending");
-        public string UnsyncedIdsFilePath = CreateFilePath(userId, deviceId, "unsynced");
-        public string ResetSyncFilePath = CreateFilePath(userId, deviceId, "reset-sync");
-
-        public readonly long LastAccessTime
-        {
-            get => long.Parse(GetMetadata("LastAccessTime") ?? "0");
-            set => SetMetadata("LastAccessTime", value.ToString());
-        }
-
-        private static string CreateFilePath(string userId, string? deviceId = null, string? metadataKey = null)
-        {
-            return Path.Join("sync", userId, deviceId, metadataKey);
-        }
-
-        private readonly string? GetMetadata(string metadataKey)
-        {
-            var path = CreateFilePath(userId, deviceId, metadataKey);
-            if (!File.Exists(path)) return null;
-            return File.ReadAllText(path);
-        }
-
-        private readonly void SetMetadata(string metadataKey, string value)
-        {
-            try
-            {
-                var path = CreateFilePath(userId, deviceId, metadataKey);
-                File.WriteAllText(path, value);
-            }
-            catch (DirectoryNotFoundException) { }
-        }
-    }
-
-    public class SyncDeviceService(SyncDevice device)
-    {
-        public string[] GetUnsyncedIds()
-        {
-            try
-            {
-                return File.ReadAllLines(device.UnsyncedIdsFilePath);
-            }
-            catch { return []; }
-        }
-
-        public string[] GetUnsyncedIds(string deviceId)
-        {
-            try
-            {
-                return File.ReadAllLines(Path.Join(device.UserSyncDirectoryPath, deviceId, "unsynced"));
-            }
-            catch { return []; }
-        }
-
-        public string[] FetchUnsyncedIds()
-        {
-            if (IsSyncReset()) return [];
-            try
-            {
-                var unsyncedIds = GetUnsyncedIds();
-                lock (device.DeviceId)
-                {
-                    if (IsSyncPending())
-                    {
-                        unsyncedIds = unsyncedIds.Union(File.ReadAllLines(device.PendingIdsFilePath)).ToArray();
-                    }
-
-                    if (unsyncedIds.Length == 0) return [];
-
-                    File.Delete(device.UnsyncedIdsFilePath);
-                    File.WriteAllLines(device.PendingIdsFilePath, unsyncedIds);
-                }
-                return unsyncedIds;
-            }
-            catch
-            {
-                return [];
-            }
-        }
-
-        public void WritePendingIds(IEnumerable<string> ids)
-        {
-            lock (device.DeviceId)
-            {
-                File.WriteAllLines(device.PendingIdsFilePath, ids);
-            }
-        }
-
-        public bool IsSyncReset()
-        {
-            return File.Exists(device.ResetSyncFilePath);
-        }
-        public bool IsSyncReset(string deviceId)
-        {
-            return File.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId, "reset-sync"));
-        }
-
-        public bool IsSyncPending()
-        {
-            return File.Exists(device.PendingIdsFilePath);
-        }
-
-        public bool IsUnsynced()
-        {
-            return File.Exists(device.UnsyncedIdsFilePath);
-        }
-
-        public void Reset()
-        {
-            try
-            {
-                lock (device.UserId)
-                {
-                    File.Delete(device.ResetSyncFilePath);
-                    File.Delete(device.PendingIdsFilePath);
-                }
-            }
-            catch (FileNotFoundException) { }
-            catch (DirectoryNotFoundException) { }
-        }
-
-        public bool IsDeviceRegistered()
-        {
-            return Directory.Exists(device.UserDeviceDirectoryPath);
-        }
-        public bool IsDeviceRegistered(string deviceId)
-        {
-            return Directory.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId));
-        }
-
-        public string[] ListDevices()
-        {
-            return Directory.GetDirectories(device.UserSyncDirectoryPath).Select((path) => path[(path.LastIndexOf(Path.DirectorySeparatorChar) + 1)..]).ToArray();
-        }
-
-        public void ResetDevices()
-        {
-            lock (device.UserId)
-            {
-                if (File.Exists(device.UserSyncDirectoryPath)) File.Delete(device.UserSyncDirectoryPath);
-                Directory.CreateDirectory(device.UserSyncDirectoryPath);
-            }
-        }
-
-        public void AddIdsToOtherDevices(List<string> ids)
-        {
-            device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-            foreach (string id in ListDevices())
-            {
-                if (id == device.DeviceId || IsSyncReset(id)) continue;
-
-                lock (id)
-                {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = GetUnsyncedIds(id);
-                    File.WriteAllLines(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds));
-                }
-            }
-        }
-
-        public void AddIdsToAllDevices(List<string> ids)
-        {
-            foreach (var id in ListDevices())
-            {
-                if (IsSyncReset(id)) return;
-                lock (id)
-                {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = GetUnsyncedIds(id);
-                    File.WriteAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds));
-                }
-            }
-        }
-
-        public void RegisterDevice()
-        {
-            lock (device.UserId)
-            {
-                if (Directory.Exists(device.UserDeviceDirectoryPath))
-                    Directory.Delete(device.UserDeviceDirectoryPath, true);
-                Directory.CreateDirectory(device.UserDeviceDirectoryPath);
-                File.Create(device.ResetSyncFilePath).Close();
-                device.LastAccessTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-            }
-        }
-
-        public void UnregisterDevice()
-        {
-            lock (device.UserId)
-            {
-                if (!Path.Exists(device.UserDeviceDirectoryPath)) return;
-                Directory.Delete(device.UserDeviceDirectoryPath, true);
-            }
-        }
-    }
-}

Notesnook.API/Services/UserService.cs

@@ -18,9 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.IO;
 using System.Net.Http;
-using System.Text;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
@@ -65,8 +63,7 @@ namespace Notesnook.API.Services
             if (!response.Success || (response.Errors != null && response.Errors.Length > 0))
             {
                 await Slogger<UserService>.Error(nameof(CreateUserAsync), "Couldn't sign up.", JsonSerializer.Serialize(response));
-                if (response.Errors != null && response.Errors.Length > 0)
-                    throw new Exception(string.Join(" ", response.Errors));
+                if (response.Errors != null && response.Errors.Length > 0) throw new Exception(string.Join(" ", response.Errors));
                 else throw new Exception("Could not create a new account. Error code: " + response.StatusCode);
             }
 
@@ -79,7 +76,7 @@ namespace Notesnook.API.Services
 
             if (!Constants.IS_SELF_HOSTED)
             {
-                await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.CreateSubscriptionTopic, new CreateSubscriptionMessage
+                await WampServers.SubscriptionServer.PublishMessageAsync(WampServers.SubscriptionServer.Topics.CreateSubscriptionTopic, new CreateSubscriptionMessage
                 {
                     AppId = ApplicationType.NOTESNOOK,
                     Provider = SubscriptionProvider.STREETWRITERS,
@@ -92,11 +89,10 @@ namespace Notesnook.API.Services
             await Slogger<UserService>.Info(nameof(CreateUserAsync), "New user created.", JsonSerializer.Serialize(response));
         }
 
-        public async Task<UserResponse> GetUserAsync(string userId)
+        public async Task<UserResponse> GetUserAsync(bool repair = true)
         {
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
-
-            var user = await userService.GetUserAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User not found.");
+            UserResponse response = await httpClient.ForwardAsync<UserResponse>(this.HttpContextAccessor, $"{Servers.IdentityServer.ToString()}/account", HttpMethod.Get);
+            if (!response.Success) return response;
 
             ISubscription subscription = null;
             if (Constants.IS_SELF_HOSTED)
@@ -106,7 +102,7 @@ namespace Notesnook.API.Services
                     AppId = ApplicationType.NOTESNOOK,
                     Provider = SubscriptionProvider.STREETWRITERS,
                     Type = SubscriptionType.PREMIUM,
-                    UserId = user.UserId,
+                    UserId = response.UserId,
                     StartDate = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
                     // this date doesn't matter as the subscription is static.
                     ExpiryDate = DateTimeOffset.UtcNow.AddYears(1).ToUnixTimeMilliseconds()
@@ -114,48 +110,61 @@ namespace Notesnook.API.Services
             }
             else
             {
-                var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
-                subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId);
+                SubscriptionResponse subscriptionResponse = await httpClient.ForwardAsync<SubscriptionResponse>(this.HttpContextAccessor, $"{Servers.SubscriptionServer}/subscriptions", HttpMethod.Get);
+                if (repair && subscriptionResponse.StatusCode == 404)
+                {
+                    await Slogger<UserService>.Error(nameof(GetUserAsync), "Repairing user subscription.", JsonSerializer.Serialize(response));
+                    // user was partially created. We should continue the process here.
+                    await WampServers.SubscriptionServer.PublishMessageAsync(WampServers.SubscriptionServer.Topics.CreateSubscriptionTopic, new CreateSubscriptionMessage
+                    {
+                        AppId = ApplicationType.NOTESNOOK,
+                        Provider = SubscriptionProvider.STREETWRITERS,
+                        Type = SubscriptionType.TRIAL,
+                        UserId = response.UserId,
+                        StartTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                        ExpiryTime = DateTimeOffset.UtcNow.AddDays(7).ToUnixTimeMilliseconds()
+                    });
+                    // just a dummy object
+                    subscriptionResponse.Subscription = new Subscription
+                    {
+                        AppId = ApplicationType.NOTESNOOK,
+                        Provider = SubscriptionProvider.STREETWRITERS,
+                        Type = SubscriptionType.TRIAL,
+                        UserId = response.UserId,
+                        StartDate = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                        ExpiryDate = DateTimeOffset.UtcNow.AddDays(7).ToUnixTimeMilliseconds()
+                    };
+                }
+                subscription = subscriptionResponse.Subscription;
             }
 
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == user.UserId) ?? throw new Exception("User settings not found.");
-            return new UserResponse
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == response.UserId);
+            if (repair && userSettings == null)
             {
-                UserId = user.UserId,
-                Email = user.Email,
-                IsEmailConfirmed = user.IsEmailConfirmed,
-                MarketingConsent = user.MarketingConsent,
-                MFA = user.MFA,
-                PhoneNumber = user.PhoneNumber,
-                AttachmentsKey = userSettings.AttachmentsKey,
-                MonographPasswordsKey = userSettings.MonographPasswordsKey,
-                Salt = userSettings.Salt,
-                Subscription = subscription,
-                Success = true,
-                StatusCode = 200
-            };
+                await Slogger<UserService>.Error(nameof(GetUserAsync), "Repairing user settings.", JsonSerializer.Serialize(response));
+                userSettings = new UserSettings
+                {
+                    UserId = response.UserId,
+                    LastSynced = 0,
+                    Salt = GetSalt()
+                };
+                await Repositories.UsersSettings.InsertAsync(userSettings);
+            }
+            response.AttachmentsKey = userSettings.AttachmentsKey;
+            response.Salt = userSettings.Salt;
+            response.Subscription = subscription;
+            return response;
         }
 
-        public async Task SetUserKeysAsync(string userId, UserKeys keys)
+        public async Task SetUserAttachmentsKeyAsync(string userId, IEncrypted key)
         {
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId) ?? throw new Exception("User not found.");
-
-            if (keys.AttachmentsKey != null)
-            {
-                userSettings.AttachmentsKey = keys.AttachmentsKey;
-            }
-            if (keys.MonographPasswordsKey != null)
-            {
-                userSettings.MonographPasswordsKey = keys.MonographPasswordsKey;
-            }
-
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+            userSettings.AttachmentsKey = (EncryptedData)key;
             await Repositories.UsersSettings.UpdateAsync(userSettings.Id, userSettings);
         }
 
-        public async Task DeleteUserAsync(string userId)
+        public async Task<bool> DeleteUserAsync(string userId, string jti)
         {
-            new SyncDeviceService(new SyncDevice(userId, userId)).ResetDevices();
-
             var cc = new CancellationTokenSource();
 
             Repositories.Notes.DeleteByUserId(userId);
@@ -163,59 +172,40 @@ namespace Notesnook.API.Services
             Repositories.Shortcuts.DeleteByUserId(userId);
             Repositories.Contents.DeleteByUserId(userId);
             Repositories.Settings.DeleteByUserId(userId);
-            Repositories.LegacySettings.DeleteByUserId(userId);
             Repositories.Attachments.DeleteByUserId(userId);
             Repositories.Reminders.DeleteByUserId(userId);
             Repositories.Relations.DeleteByUserId(userId);
-            Repositories.Colors.DeleteByUserId(userId);
-            Repositories.Tags.DeleteByUserId(userId);
-            Repositories.Vaults.DeleteByUserId(userId);
             Repositories.UsersSettings.Delete((u) => u.UserId == userId);
             Repositories.Monographs.DeleteMany((m) => m.UserId == userId);
 
-            var result = await unit.Commit();
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "User data deleted", userId, result.ToString());
-            if (!result) throw new Exception("Could not delete user data.");
-
             if (!Constants.IS_SELF_HOSTED)
             {
-                await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.DeleteSubscriptionTopic, new DeleteSubscriptionMessage
+                await WampServers.SubscriptionServer.PublishMessageAsync(WampServers.SubscriptionServer.Topics.DeleteSubscriptionTopic, new DeleteSubscriptionMessage
                 {
                     AppId = ApplicationType.NOTESNOOK,
                     UserId = userId
                 });
             }
 
-            await S3Service.DeleteDirectoryAsync(userId);
-        }
-
-        public async Task DeleteUserAsync(string userId, string jti, string password)
-        {
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "Deleting user account", userId);
-
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
-            await userService.DeleteUserAsync(Clients.Notesnook.Id, userId, password);
-
-            await DeleteUserAsync(userId);
-
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+            await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
             {
                 SendToAll = false,
                 OriginTokenId = jti,
                 UserId = userId,
                 Message = new Message
                 {
-                    Type = "logout",
-                    Data = JsonSerializer.Serialize(new { reason = "Account deleted." })
+                    Type = "userDeleted",
+                    Data = JsonSerializer.Serialize(new { reason = "accountDeleted" })
                 }
             });
 
+            await S3Service.DeleteDirectoryAsync(userId);
+
+            return await unit.Commit();
         }
 
         public async Task<bool> ResetUserAsync(string userId, bool removeAttachments)
         {
-            new SyncDeviceService(new SyncDevice(userId, userId)).ResetDevices();
-
             var cc = new CancellationTokenSource();
 
             Repositories.Notes.DeleteByUserId(userId);
@@ -223,20 +213,15 @@ namespace Notesnook.API.Services
             Repositories.Shortcuts.DeleteByUserId(userId);
             Repositories.Contents.DeleteByUserId(userId);
             Repositories.Settings.DeleteByUserId(userId);
-            Repositories.LegacySettings.DeleteByUserId(userId);
             Repositories.Attachments.DeleteByUserId(userId);
             Repositories.Reminders.DeleteByUserId(userId);
             Repositories.Relations.DeleteByUserId(userId);
-            Repositories.Colors.DeleteByUserId(userId);
-            Repositories.Tags.DeleteByUserId(userId);
-            Repositories.Vaults.DeleteByUserId(userId);
             Repositories.Monographs.DeleteMany((m) => m.UserId == userId);
             if (!await unit.Commit()) return false;
 
             var userSettings = await Repositories.UsersSettings.FindOneAsync((s) => s.UserId == userId);
 
             userSettings.AttachmentsKey = null;
-            userSettings.MonographPasswordsKey = null;
             userSettings.VaultKey = null;
             userSettings.LastSynced = 0;
 
@@ -248,7 +233,7 @@ namespace Notesnook.API.Services
             return true;
         }
 
-        private static string GetSalt()
+        private string GetSalt()
         {
             byte[] salt = new byte[16];
             Rng.GetNonZeroBytes(salt);

Notesnook.API/Startup.cs

@@ -34,7 +34,6 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Connections;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.ResponseCompression;
-using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -51,8 +50,6 @@ using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Repositories;
 using Notesnook.API.Services;
-using OpenTelemetry.Metrics;
-using OpenTelemetry.Resources;
 using Streetwriters.Common;
 using Streetwriters.Common.Extensions;
 using Streetwriters.Common.Messages;
@@ -76,11 +73,12 @@ namespace Notesnook.API
         // This method gets called by the runtime. Use this method to add services to the container.
         public void ConfigureServices(IServiceCollection services)
         {
-            services.AddSingleton(MongoDbContext.CreateMongoDbClient(new DbSettings
+            var dbSettings = new DbSettings
             {
                 ConnectionString = Constants.MONGODB_CONNECTION_STRING,
                 DatabaseName = Constants.MONGODB_DATABASE_NAME
-            }));
+            };
+            services.AddSingleton<IDbSettings>(dbSettings);
 
             services.TryAddSingleton<IHttpContextAccessor, HttpContextAccessor>();
 
@@ -108,13 +106,23 @@ namespace Notesnook.API
                     policy.RequireAuthenticatedUser();
                     policy.Requirements.Add(new SyncRequirement());
                 });
+                options.AddPolicy("Verified", policy =>
+                {
+                    policy.AuthenticationSchemes.Add("introspection");
+                    policy.RequireAuthenticatedUser();
+                    policy.Requirements.Add(new EmailVerifiedRequirement());
+                });
                 options.AddPolicy("Pro", policy =>
                 {
                     policy.AuthenticationSchemes.Add("introspection");
                     policy.RequireAuthenticatedUser();
-                    policy.Requirements.Add(new SyncRequirement());
                     policy.Requirements.Add(new ProUserRequirement());
                 });
+                options.AddPolicy("BasicAdmin", policy =>
+                {
+                    policy.AuthenticationSchemes.Add("BasicAuthentication");
+                    policy.RequireClaim(ClaimTypes.Role, "Admin");
+                });
 
                 options.DefaultPolicy = options.GetPolicy("Notesnook");
             }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>(); ;
@@ -144,55 +152,48 @@ namespace Notesnook.API
                     context.HttpContext.User = context.Principal;
                     return Task.CompletedTask;
                 };
-                options.CacheKeyGenerator = (options, token) => (token + ":" + "reference_token").Sha256();
                 options.SaveToken = true;
                 options.EnableCaching = true;
                 options.CacheDuration = TimeSpan.FromMinutes(30);
             });
 
-            // Serializer.RegisterSerializer(new SyncItemBsonSerializer());
             if (!BsonClassMap.IsClassMapRegistered(typeof(UserSettings)))
+            {
                 BsonClassMap.RegisterClassMap<UserSettings>();
+            }
 
             if (!BsonClassMap.IsClassMapRegistered(typeof(EncryptedData)))
+            {
                 BsonClassMap.RegisterClassMap<EncryptedData>();
+            }
 
             if (!BsonClassMap.IsClassMapRegistered(typeof(CallToAction)))
+            {
                 BsonClassMap.RegisterClassMap<CallToAction>();
+            }
+
+            if (!BsonClassMap.IsClassMapRegistered(typeof(Announcement)))
+            {
+                BsonClassMap.RegisterClassMap<Announcement>();
+            }
 
             services.AddScoped<IDbContext, MongoDbContext>();
             services.AddScoped<IUnitOfWork, UnitOfWork>();
+            services.AddScoped(typeof(Repository<>));
+            services.AddScoped(typeof(SyncItemsRepository<>));
 
-            services.AddRepository<UserSettings>("user_settings", "notesnook")
-                    .AddRepository<Monograph>("monographs", "notesnook")
-                    .AddRepository<Announcement>("announcements", "notesnook");
-
-            services.AddMongoCollection(Collections.SettingsKey)
-                    .AddMongoCollection(Collections.AttachmentsKey)
-                    .AddMongoCollection(Collections.ContentKey)
-                    .AddMongoCollection(Collections.NotesKey)
-                    .AddMongoCollection(Collections.NotebooksKey)
-                    .AddMongoCollection(Collections.RelationsKey)
-                    .AddMongoCollection(Collections.RemindersKey)
-                    .AddMongoCollection(Collections.LegacySettingsKey)
-                    .AddMongoCollection(Collections.ShortcutsKey)
-                    .AddMongoCollection(Collections.TagsKey)
-                    .AddMongoCollection(Collections.ColorsKey)
-                    .AddMongoCollection(Collections.VaultsKey);
-
-            services.AddScoped<ISyncItemsRepositoryAccessor, SyncItemsRepositoryAccessor>();
-            services.AddScoped<IUserService, UserService>();
-            services.AddScoped<IS3Service, S3Service>();
+            services.TryAddTransient<ISyncItemsRepositoryAccessor, SyncItemsRepositoryAccessor>();
+            services.TryAddTransient<IUserService, UserService>();
+            services.TryAddTransient<IS3Service, S3Service>();
 
             services.AddControllers();
 
-            services.AddHealthChecks(); // .AddMongoDb(dbSettings.ConnectionString, dbSettings.DatabaseName, "database-check");
+            services.AddHealthChecks().AddMongoDb(dbSettings.ConnectionString, dbSettings.DatabaseName, "database-check");
             services.AddSignalR((hub) =>
             {
                 hub.MaximumReceiveMessageSize = 100 * 1024 * 1024;
-                hub.ClientTimeoutInterval = TimeSpan.FromMinutes(10);
                 hub.EnableDetailedErrors = true;
-            }).AddMessagePackProtocol().AddJsonProtocol();
+            }).AddMessagePackProtocol();
 
             services.AddResponseCompression(options =>
             {
@@ -209,13 +210,6 @@ namespace Notesnook.API
             {
                 options.Level = CompressionLevel.Fastest;
             });
-
-            services.AddOpenTelemetry()
-                    .ConfigureResource(resource => resource
-                        .AddService(serviceName: "Notesnook.API"))
-                    .WithMetrics((builder) => builder
-                            .AddMeter("Notesnook.API.Metrics.Sync")
-                            .AddPrometheusExporter());
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
@@ -229,24 +223,17 @@ namespace Notesnook.API
                 });
             }
 
-            app.UseOpenTelemetryPrometheusScrapingEndpoint((context) => context.Request.Path == "/metrics" && context.Connection.LocalPort == 5067);
             app.UseResponseCompression();
 
             app.UseCors("notesnook");
-            app.UseVersion(Servers.NotesnookAPI);
+            app.UseVersion();
 
             app.UseWamp(WampServers.NotesnookServer, (realm, server) =>
             {
-                realm.Subscribe<DeleteUserMessage>(IdentityServerTopics.DeleteUserTopic, async (ev) =>
+                IUserService service = app.GetScopedService<IUserService>();
+                realm.Subscribe<DeleteUserMessage>(server.Topics.DeleteUserTopic, async (ev) =>
                 {
-                    IUserService service = app.GetScopedService<IUserService>();
-                    await service.DeleteUserAsync(ev.UserId);
-                });
-
-                realm.Subscribe<ClearCacheMessage>(IdentityServerTopics.ClearCacheTopic, (ev) =>
-                {
-                    IDistributedCache cache = app.GetScopedService<IDistributedCache>();
-                    ev.Keys.ForEach((key) => cache.Remove(key));
+                    await service.DeleteUserAsync(ev.UserId, null);
                 });
             });
 
@@ -257,7 +244,6 @@ namespace Notesnook.API
 
             app.UseEndpoints(endpoints =>
             {
-                endpoints.MapPrometheusScrapingEndpoint();
                 endpoints.MapControllers();
                 endpoints.MapHealthChecks("/health");
                 endpoints.MapHub<SyncHub>("/hubs/sync", options =>
@@ -265,21 +251,7 @@ namespace Notesnook.API
                     options.CloseOnAuthenticationExpiration = false;
                     options.Transports = HttpTransportType.WebSockets;
                 });
-                endpoints.MapHub<SyncV2Hub>("/hubs/sync/v2", options =>
-                {
-                    options.CloseOnAuthenticationExpiration = false;
-                    options.Transports = HttpTransportType.WebSockets;
-                });
             });
         }
     }
-
-    public static class ServiceCollectionMongoCollectionExtensions
-    {
-        public static IServiceCollection AddMongoCollection(this IServiceCollection services, string collectionName, string database = "notesnook")
-        {
-            services.AddKeyedSingleton(collectionName, (provider, key) => MongoDbContext.GetMongoCollection<SyncItem>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
-            return services;
-        }
-    }
 }

Notesnook.API/appsettings.Development.json

@@ -3,9 +3,7 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information",
-      "Microsoft.AspNetCore.SignalR": "Trace",
-      "Microsoft.AspNetCore.Http.Connections": "Trace"
+      "Microsoft.Hosting.Lifetime": "Information"
     }
   },
   "MongoDbSettings": {

Notesnook.API/appsettings.json

@@ -1,7 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Warning"
-    }
-  }
-}

README.md

@@ -8,7 +8,7 @@ This repo contains the full source code of the Notesnook Sync Server licensed un
 
 Requirements:
 
-1. [.NET 8](https://dotnet.microsoft.com/en-us/download/dotnet/8.0)
+1. [.NET 7](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
 2. [git](https://git-scm.com/downloads)
 
 The first step is to `clone` the repository:
@@ -55,30 +55,35 @@ dotnet run --project Streetwriters.Identity/Streetwriters.Identity.csproj
 
 The sync server can easily be started using Docker.
 
+The first step is to `clone` the repository:
+
 ```bash
-wget https://raw.githubusercontent.com/streetwriters/notesnook-sync-server/master/docker-compose.yml
+git clone https://github.com/streetwriters/notesnook-sync-server.git
+
+# change directory
+cd notesnook-sync-server
 ```
 
 And then use Docker Compose to start the servers:
 
 ```bash
-docker compose up
+docker-compose up
 ```
 
 This takes care of setting up everything including MongoDB, Minio etc.
 
 ## TODO Self-hosting
 
-**Note: Self-hosting the Notesnook Sync Server is now possible, but without support. Documentation will be provided at a later date. We are working to enable full on-premise self-hosting, so stay tuned!**
+**Note: Self-hosting the Notesnook Sync Server is not yet possible. We are working to enable full on-premise self hosting so stay tuned!**
 
 - [x] Open source the Sync server
 - [x] Open source the Identity server
 - [x] Open source the SSE Messaging infrastructure
 - [x] Fully Dockerize all services
 - [x] Use self-hosted Minio for S3 storage
-- [x] Publish on DockerHub
-- [x] Add settings to change server URLs in Notesnook client apps (starting from v3.0.18)
+- [ ] Publish on DockerHub
 - [ ] Write self hosting docs
+- [ ] Add settings to change server URLs in Notesnook client apps
 
 ## License
 

Streetwriters.Common/Clients.cs

@@ -29,19 +29,19 @@ namespace Streetwriters.Common
 {
     public class Clients
     {
-        public static readonly Client Notesnook = new()
+        private static Client Notesnook = new Client
         {
             Id = "notesnook",
             Name = "Notesnook",
             SenderEmail = Constants.NOTESNOOK_SENDER_EMAIL,
-            SenderName = "Notesnook",
+            SenderName = Constants.NOTESNOOK_SENDER_NAME,
             Type = ApplicationType.NOTESNOOK,
             AppId = ApplicationType.NOTESNOOK,
             AccountRecoveryRedirectURL = $"{Constants.NOTESNOOK_APP_HOST}/account/recovery",
             EmailConfirmedRedirectURL = $"{Constants.NOTESNOOK_APP_HOST}/account/verified",
             OnEmailConfirmed = async (userId) =>
             {
-                await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+                await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
                 {
                     UserId = userId,
                     Message = new Message
@@ -53,7 +53,7 @@ namespace Streetwriters.Common
             }
         };
 
-        public static Dictionary<string, Client> ClientsMap = new()
+        public static Dictionary<string, Client> ClientsMap = new Dictionary<string, Client>
         {
             { "notesnook", Notesnook }
         };

Streetwriters.Common/Constants.cs

@@ -23,57 +23,56 @@ namespace Streetwriters.Common
 {
     public class Constants
     {
-        public static int COMPATIBILITY_VERSION = 1;
         public static bool IS_SELF_HOSTED => Environment.GetEnvironmentVariable("SELF_HOSTED") == "1";
-        public static bool DISABLE_SIGNUPS => Environment.GetEnvironmentVariable("DISABLE_SIGNUPS") == "true";
-        public static string INSTANCE_NAME => Environment.GetEnvironmentVariable("INSTANCE_NAME") ?? "default";
 
         // S3 related
         public static string S3_ACCESS_KEY => Environment.GetEnvironmentVariable("S3_ACCESS_KEY");
         public static string S3_ACCESS_KEY_ID => Environment.GetEnvironmentVariable("S3_ACCESS_KEY_ID");
         public static string S3_SERVICE_URL => Environment.GetEnvironmentVariable("S3_SERVICE_URL");
         public static string S3_REGION => Environment.GetEnvironmentVariable("S3_REGION");
-        public static string S3_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_BUCKET_NAME");
-        public static string S3_INTERNAL_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_INTERNAL_BUCKET_NAME");
-        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
 
         // SMTP settings
         public static string SMTP_USERNAME => Environment.GetEnvironmentVariable("SMTP_USERNAME");
         public static string SMTP_PASSWORD => Environment.GetEnvironmentVariable("SMTP_PASSWORD");
         public static string SMTP_HOST => Environment.GetEnvironmentVariable("SMTP_HOST");
         public static string SMTP_PORT => Environment.GetEnvironmentVariable("SMTP_PORT");
+        public static string SMTP_REPLYTO_NAME => Environment.GetEnvironmentVariable("SMTP_REPLYTO_NAME");
         public static string SMTP_REPLYTO_EMAIL => Environment.GetEnvironmentVariable("SMTP_REPLYTO_EMAIL");
-        public static string NOTESNOOK_SENDER_EMAIL => Environment.GetEnvironmentVariable("NOTESNOOK_SENDER_EMAIL") ?? Environment.GetEnvironmentVariable("SMTP_USERNAME");
+        public static string NOTESNOOK_SENDER_EMAIL => Environment.GetEnvironmentVariable("NOTESNOOK_SENDER_EMAIL");
+        public static string NOTESNOOK_SENDER_NAME => Environment.GetEnvironmentVariable("NOTESNOOK_SENDER_NAME");
 
         public static string NOTESNOOK_APP_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_APP_HOST");
         public static string NOTESNOOK_API_SECRET => Environment.GetEnvironmentVariable("NOTESNOOK_API_SECRET");
 
         // MessageBird is used for SMS sending
-        public static string TWILIO_ACCOUNT_SID => Environment.GetEnvironmentVariable("TWILIO_ACCOUNT_SID");
-        public static string TWILIO_AUTH_TOKEN => Environment.GetEnvironmentVariable("TWILIO_AUTH_TOKEN");
-        public static string TWILIO_SERVICE_SID => Environment.GetEnvironmentVariable("TWILIO_SERVICE_SID");
+        public static string MESSAGEBIRD_ACCESS_KEY => Environment.GetEnvironmentVariable("MESSAGEBIRD_ACCESS_KEY");
+
         // Server discovery
-        public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT") ?? "80");
+        public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT"));
         public static string NOTESNOOK_SERVER_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_HOST");
+        public static string NOTESNOOK_SERVER_DOMAIN => Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_DOMAIN");
         public static string NOTESNOOK_CERT_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_PATH");
         public static string NOTESNOOK_CERT_KEY_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_KEY_PATH");
 
-        public static int IDENTITY_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("IDENTITY_SERVER_PORT") ?? "80");
+        public static int IDENTITY_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("IDENTITY_SERVER_PORT"));
         public static string IDENTITY_SERVER_HOST => Environment.GetEnvironmentVariable("IDENTITY_SERVER_HOST");
-        public static Uri IDENTITY_SERVER_URL => new(Environment.GetEnvironmentVariable("IDENTITY_SERVER_URL"));
+        public static string IDENTITY_SERVER_DOMAIN => Environment.GetEnvironmentVariable("IDENTITY_SERVER_DOMAIN");
         public static string IDENTITY_CERT_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_PATH");
         public static string IDENTITY_CERT_KEY_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_KEY_PATH");
 
-        public static int SSE_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SSE_SERVER_PORT") ?? "80");
+        public static int SSE_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SSE_SERVER_PORT"));
         public static string SSE_SERVER_HOST => Environment.GetEnvironmentVariable("SSE_SERVER_HOST");
+        public static string SSE_SERVER_DOMAIN => Environment.GetEnvironmentVariable("SSE_SERVER_DOMAIN");
         public static string SSE_CERT_PATH => Environment.GetEnvironmentVariable("SSE_CERT_PATH");
         public static string SSE_CERT_KEY_PATH => Environment.GetEnvironmentVariable("SSE_CERT_KEY_PATH");
 
         // internal
         public static string MONGODB_CONNECTION_STRING => Environment.GetEnvironmentVariable("MONGODB_CONNECTION_STRING");
         public static string MONGODB_DATABASE_NAME => Environment.GetEnvironmentVariable("MONGODB_DATABASE_NAME");
-        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_PORT") ?? "80");
+        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
+        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_PORT"));
         public static string SUBSCRIPTIONS_SERVER_HOST => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_HOST");
+        public static string SUBSCRIPTIONS_SERVER_DOMAIN => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_DOMAIN");
         public static string SUBSCRIPTIONS_CERT_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_PATH");
         public static string SUBSCRIPTIONS_CERT_KEY_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_KEY_PATH");
         public static string[] NOTESNOOK_CORS_ORIGINS => Environment.GetEnvironmentVariable("NOTESNOOK_CORS")?.Split(",") ?? new string[] { };

Streetwriters.Common/Enums/SubscriptionProvider.cs

@@ -24,7 +24,6 @@ namespace Streetwriters.Common.Enums
         STREETWRITERS = 0,
         APPLE = 1,
         GOOGLE = 2,
-        PADDLE = 3,
-        GIFT_CARD = 4,
+        PADDLE = 3
     }
 }

Streetwriters.Common/Enums/SubscriptionType.cs

@@ -26,7 +26,6 @@ namespace Streetwriters.Common.Enums
         BETA = 2,
         PREMIUM = 5,
         PREMIUM_EXPIRED = 6,
-        PREMIUM_CANCELED = 7,
-        PREMIUM_PAUSED = 8
+        PREMIUM_CANCELED = 7
     }
 }

Streetwriters.Common/Extensions/AppBuilderExtensions.cs

@@ -18,8 +18,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.Collections.Generic;
-using System.Text.Json;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DependencyInjection;
@@ -32,20 +30,13 @@ namespace Streetwriters.Common.Extensions
 {
     public static class AppBuilderExtensions
     {
-        public static IApplicationBuilder UseVersion(this IApplicationBuilder app, Server server)
+        public static IApplicationBuilder UseVersion(this IApplicationBuilder app)
         {
             app.Map("/version", (app) =>
             {
                 app.Run(async context =>
                 {
-                    context.Response.ContentType = "application/json";
-                    var data = new Dictionary<string, object>
-                    {
-                        { "version", Constants.COMPATIBILITY_VERSION },
-                        { "id", server.Id },
-                        { "instance", Constants.INSTANCE_NAME }
-                    };
-                    await context.Response.WriteAsync(JsonSerializer.Serialize(data));
+                    await context.Response.WriteAsync(Version.AsString());
                 });
             });
             return app;

Streetwriters.Common/Extensions/ServiceCollectionServiceExtensions.cs

@@ -18,20 +18,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using Microsoft.Extensions.DependencyInjection;
-using Streetwriters.Data.DbContexts;
-using Streetwriters.Data.Repositories;
 
 namespace Streetwriters.Common.Extensions
 {
     public static class ServiceCollectionServiceExtensions
     {
-        public static IServiceCollection AddRepository<T>(this IServiceCollection services, string collectionName, string database) where T : class
-        {
-            services.AddSingleton((provider) => MongoDbContext.GetMongoCollection<T>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
-            services.AddScoped<Repository<T>>();
-            return services;
-        }
-
         public static IServiceCollection AddDefaultCors(this IServiceCollection services)
         {
             services.AddCors(options =>

Streetwriters.Common/Extensions/StringExtensions.cs

@@ -26,11 +26,15 @@ namespace System
 {
     public static class StringExtensions
     {
-        public static string Sha256(this string input)
+        public static string ToSha256(this string rawData, int maxLength = 12)
         {
-            var bytes = Encoding.UTF8.GetBytes(input);
-            var hash = SHA256.HashData(bytes);
-            return Convert.ToBase64String(hash);
+            // Create a SHA256   
+            using (SHA256 sha256Hash = SHA256.Create())
+            {
+                // ComputeHash - returns byte array  
+                byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(rawData));
+                return ToHex(bytes, 0, maxLength);
+            }
         }
 
         public static byte[] CompressBrotli(this string input)

Streetwriters.Common/Helpers/HtmlHelper.cs

@@ -1,23 +0,0 @@
-using System.IO;
-using WebMarkupMin.Core;
-using WebMarkupMin.Core.Loggers;
-
-namespace Streetwriters.Common.Helpers
-{
-    public static class HtmlHelper
-    {
-        public static string ReadMinifiedHtmlFile(string path)
-        {
-            var settings = new HtmlMinificationSettings()
-            {
-                WhitespaceMinificationMode = WhitespaceMinificationMode.Medium,
-            };
-            var cssMinifier = new KristensenCssMinifier();
-            var jsMinifier = new CrockfordJsMinifier();
-
-            var minifier = new HtmlMinifier(settings, cssMinifier, jsMinifier, new NullLogger());
-
-            return minifier.Minify(File.ReadAllText(path), false).MinifiedContent;
-        }
-    }
-}

Streetwriters.Common/Helpers/WampHelper.cs

@@ -27,9 +27,9 @@ namespace Streetwriters.Common.Helpers
 {
     public class WampHelper
     {
-        public static async Task<IWampRealmProxy> OpenWampChannelAsync(string server, string realmName)
+        public static async Task<IWampRealmProxy> OpenWampChannelAsync<T>(string server, string realmName)
         {
-            DefaultWampChannelFactory channelFactory = new();
+            DefaultWampChannelFactory channelFactory = new DefaultWampChannelFactory();
 
             IWampChannel channel = channelFactory.CreateJsonChannel(server, realmName);
 

Streetwriters.Common/Interfaces/IEmailSender.cs

@@ -1,19 +0,0 @@
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using MimeKit;
-using MimeKit.Cryptography;
-using Streetwriters.Common.Models;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IEmailSender
-    {
-        Task SendEmailAsync(
-            string email,
-            EmailTemplate template,
-            IClient client,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
-        );
-    }
-}

Streetwriters.Common/Interfaces/IUserAccountService.cs

@@ -1,16 +0,0 @@
-using System.Threading.Tasks;
-using Streetwriters.Common.Models;
-using WampSharp.V2.Rpc;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IUserAccountService
-    {
-        [WampProcedure("co.streetwriters.identity.users.get_user")]
-        Task<UserModel> GetUserAsync(string clientId, string userId);
-        [WampProcedure("co.streetwriters.identity.users.delete_user")]
-        Task DeleteUserAsync(string clientId, string userId, string password);
-        // [WampProcedure("co.streetwriters.identity.users.create_user")]
-        // Task<UserModel> CreateUserAsync();
-    }
-}

Streetwriters.Common/Interfaces/IUserSubscriptionService.cs

@@ -1,13 +0,0 @@
-using System.Threading.Tasks;
-using Streetwriters.Common.Helpers;
-using Streetwriters.Common.Models;
-using WampSharp.V2.Rpc;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IUserSubscriptionService
-    {
-        [WampProcedure("co.streetwriters.subscriptions.subscriptions.get_user_subscription")]
-        Task<Subscription> GetUserSubscriptionAsync(string clientId, string userId);
-    }
-}

Streetwriters.Common/Models/EmailTemplate.cs

@@ -1,11 +0,0 @@
-namespace Streetwriters.Common.Models
-{
-    public class EmailTemplate
-    {
-        public int? Id { get; set; }
-        public object Data { get; set; }
-        public string Subject { get; set; }
-        public string Html { get; set; }
-        public string Text { get; set; }
-    }
-}

Streetwriters.Common/Models/GiftCard.cs

@@ -1,29 +0,0 @@
-using System.Text.Json.Serialization;
-using MongoDB.Bson;
-using MongoDB.Bson.Serialization.Attributes;
-using Streetwriters.Common.Interfaces;
-
-namespace Streetwriters.Common.Models
-{
-    public class GiftCard : IDocument
-    {
-        public GiftCard()
-        {
-            Id = ObjectId.GenerateNewId().ToString();
-        }
-
-        public string Code { get; set; }
-        public string OrderId { get; set; }
-        public string OrderIdType { get; set; }
-        public string ProductId { get; set; }
-        public string RedeemedBy { get; set; }
-        public long RedeemedAt { get; set; }
-        public long Timestamp { get; set; }
-        public long Term { get; set; }
-
-        [BsonId]
-        [BsonRepresentation(BsonType.ObjectId)]
-        [JsonIgnore]
-        public string Id { get; set; }
-    }
-}

Streetwriters.Common/Models/Offer.cs

@@ -26,9 +26,11 @@ using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("subscriptions", "offers")]
     public class Offer : IOffer
     {
         public Offer()

Streetwriters.Common/Models/Role.cs

@@ -20,9 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
 using AspNetCore.Identity.Mongo.Model;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("identity", "roles")]
     public class Role : MongoRole
     {
         //     [DataMember(Name = "email")]

Streetwriters.Common/Models/Subscription.cs

@@ -24,9 +24,11 @@ using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("subscriptions", "subscriptions")]
     public class Subscription : ISubscription
     {
         public Subscription()

Streetwriters.Common/Models/User.cs

@@ -20,9 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
 using AspNetCore.Identity.Mongo.Model;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("identity", "users")]
     public class User : MongoUser
     {
     }

Streetwriters.Common/Models/UserModel.cs

@@ -35,9 +35,6 @@ namespace Streetwriters.Common.Models
         [JsonPropertyName("isEmailConfirmed")]
         public bool IsEmailConfirmed { get; set; }
 
-        [JsonPropertyName("marketingConsent")]
-        public bool MarketingConsent { get; set; }
-
         [JsonPropertyName("mfa")]
         public MFAConfig MFA { get; set; }
     }

Streetwriters.Common/Servers.cs

@@ -17,7 +17,6 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System;
 using System.Net.NetworkInformation;
 using System.Net.Sockets;
 using System.Security.Cryptography.X509Certificates;
@@ -35,10 +34,10 @@ namespace Streetwriters.Common
             if (!string.IsNullOrEmpty(originCertPath) && !string.IsNullOrEmpty(originCertKeyPath))
                 this.SSLCertificate = X509Certificate2.CreateFromPemFile(originCertPath, originCertKeyPath);
         }
-        public string Id { get; set; }
+
         public int Port { get; set; }
         public string Hostname { get; set; }
-        public Uri PublicURL { get; set; }
+        public string Domain { get; set; }
         public X509Certificate2 SSLCertificate { get; }
         public bool IsSecure { get => this.SSLCertificate != null; }
 
@@ -63,14 +62,14 @@ namespace Streetwriters.Common
 
     public class Servers
     {
-#if (DEBUG || STAGING)
-        public static string GetLocalIPv4()
+#if DEBUG
+        public static string GetLocalIPv4(NetworkInterfaceType _type)
         {
             var interfaces = NetworkInterface.GetAllNetworkInterfaces();
             string output = "";
             foreach (NetworkInterface item in interfaces)
             {
-                if ((item.NetworkInterfaceType == NetworkInterfaceType.Ethernet || item.NetworkInterfaceType == NetworkInterfaceType.Wireless80211) && item.OperationalStatus == OperationalStatus.Up)
+                if (item.NetworkInterfaceType == _type && item.OperationalStatus == OperationalStatus.Up)
                 {
                     foreach (UnicastIPAddressInformation ip in item.GetIPProperties().UnicastAddresses)
                     {
@@ -83,40 +82,40 @@ namespace Streetwriters.Common
             }
             return output;
         }
-        public readonly static string HOST = GetLocalIPv4();
+        public readonly static string HOST = GetLocalIPv4(NetworkInterfaceType.Ethernet);
         public static Server S3Server { get; } = new()
         {
             Port = 4568,
-            Hostname = HOST
+            Hostname = HOST,
+            Domain = HOST
         };
 #endif
         public static Server NotesnookAPI { get; } = new(Constants.NOTESNOOK_CERT_PATH, Constants.NOTESNOOK_CERT_KEY_PATH)
         {
+            Domain = Constants.NOTESNOOK_SERVER_DOMAIN,
             Port = Constants.NOTESNOOK_SERVER_PORT,
             Hostname = Constants.NOTESNOOK_SERVER_HOST,
-            Id = "notesnook-sync"
         };
 
         public static Server MessengerServer { get; } = new(Constants.SSE_CERT_PATH, Constants.SSE_CERT_KEY_PATH)
         {
+            Domain = Constants.SSE_SERVER_DOMAIN,
             Port = Constants.SSE_SERVER_PORT,
             Hostname = Constants.SSE_SERVER_HOST,
-            Id = "sse"
         };
 
         public static Server IdentityServer { get; } = new(Constants.IDENTITY_CERT_PATH, Constants.IDENTITY_CERT_KEY_PATH)
         {
-            PublicURL = Constants.IDENTITY_SERVER_URL,
+            Domain = Constants.IDENTITY_SERVER_DOMAIN,
             Port = Constants.IDENTITY_SERVER_PORT,
             Hostname = Constants.IDENTITY_SERVER_HOST,
-            Id = "auth"
         };
 
         public static Server SubscriptionServer { get; } = new(Constants.SUBSCRIPTIONS_CERT_PATH, Constants.SUBSCRIPTIONS_CERT_KEY_PATH)
         {
+            Domain = Constants.SUBSCRIPTIONS_SERVER_DOMAIN,
             Port = Constants.SUBSCRIPTIONS_SERVER_PORT,
             Hostname = Constants.SUBSCRIPTIONS_SERVER_HOST,
-            Id = "subscription"
         };
     }
 }

Streetwriters.Common/Services/EmailSender.cs

@@ -1,134 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text;
-using System.Threading.Tasks;
-using MailKit.Net.Smtp;
-using MimeKit;
-using MimeKit.Cryptography;
-using Org.BouncyCastle.Bcpg;
-using Scriban;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
-
-namespace Streetwriters.Common.Services
-{
-    public class EmailSender : IEmailSender, IAsyncDisposable
-    {
-        private readonly SmtpClient mailClient = new();
-
-        public async Task SendEmailAsync(
-            string email,
-            EmailTemplate template,
-            IClient client,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
-        )
-        {
-            if (!mailClient.IsConnected)
-            {
-                if (int.TryParse(Common.Constants.SMTP_PORT, out int port))
-                {
-                    await mailClient.ConnectAsync(
-                        Common.Constants.SMTP_HOST,
-                        port,
-                        MailKit.Security.SecureSocketOptions.Auto
-                    );
-                }
-                else
-                {
-                    throw new InvalidDataException("SMTP_PORT is not a valid integer value.");
-                }
-            }
-
-            if (!mailClient.IsAuthenticated)
-                await mailClient.AuthenticateAsync(
-                    Common.Constants.SMTP_USERNAME,
-                    Common.Constants.SMTP_PASSWORD
-                );
-
-            var message = new MimeMessage();
-            var sender = new MailboxAddress(client.SenderName, client.SenderEmail);
-            message.From.Add(sender);
-            message.To.Add(new MailboxAddress("", email));
-            message.Subject = await Template.Parse(template.Subject).RenderAsync(template.Data);
-
-            if (!string.IsNullOrEmpty(Common.Constants.SMTP_REPLYTO_EMAIL))
-                message.ReplyTo.Add(MailboxAddress.Parse(Common.Constants.SMTP_REPLYTO_EMAIL));
-
-            message.Body = await GetEmailBodyAsync(
-                template,
-                client,
-                sender,
-                gpgContext,
-                attachments
-            );
-
-            await mailClient.SendAsync(message);
-        }
-
-        private static async Task<MimeEntity> GetEmailBodyAsync(
-            EmailTemplate template,
-            IClient client,
-            MailboxAddress sender,
-            GnuPGContext gpgContext = null,
-            Dictionary<string, byte[]> attachments = null
-        )
-        {
-            var builder = new BodyBuilder();
-            try
-            {
-                builder.TextBody = await Template.Parse(template.Text).RenderAsync(template.Data);
-                builder.HtmlBody = await Template.Parse(template.Html).RenderAsync(template.Data);
-
-                if (attachments != null)
-                {
-                    foreach (var attachment in attachments)
-                    {
-                        builder.Attachments.Add(attachment.Key, attachment.Value);
-                    }
-                }
-
-                var key = gpgContext?.GetSigningKey(sender);
-                if (key != null)
-                {
-                    using (MemoryStream outputStream = new())
-                    {
-                        using (Stream armoredStream = new ArmoredOutputStream(outputStream))
-                        {
-                            key.PublicKey.Encode(armoredStream);
-                        }
-                        outputStream.Seek(0, SeekOrigin.Begin);
-                        builder.Attachments.Add(
-                            $"{client.Id}_pub.asc",
-                            Encoding.ASCII.GetBytes(
-                                Encoding.ASCII.GetString(outputStream.ToArray())
-                            )
-                        );
-                    }
-                    return await MultipartSigned.CreateAsync(
-                        gpgContext,
-                        sender,
-                        DigestAlgorithm.Sha256,
-                        builder.ToMessageBody()
-                    );
-                }
-                else
-                {
-                    return builder.ToMessageBody();
-                }
-            }
-            catch (Exception ex)
-            {
-                await Slogger<EmailSender>.Error("GetEmailBodyAsync", ex.ToString());
-                return builder.ToMessageBody();
-            }
-        }
-
-        async ValueTask IAsyncDisposable.DisposeAsync()
-        {
-            await mailClient.DisconnectAsync(true);
-            mailClient.Dispose();
-        }
-    }
-}

Streetwriters.Common/Streetwriters.Common.csproj

@@ -1,21 +1,18 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
   
   <ItemGroup>
-    <PackageReference Include="MailKit" Version="4.9.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Cors" Version="2.2.0" />
-    <PackageReference Include="Scriban" Version="5.12.1" />
     <PackageReference Include="WampSharp.Default" Version="20.1.1" />
     <PackageReference Include="WampSharp.AspNetCore.WebSockets.Server" Version="20.1.1" />
     <PackageReference Include="WampSharp.NewtonsoftMsgpack" Version="20.1.1" />
     <PackageReference Include="Microsoft.AspNetCore.WebSockets" Version="2.2.1" />
     <PackageReference Include="AspNetCore.Identity.Mongo" Version="8.3.3" />
-    <PackageReference Include="WebMarkupMin.NUglify" Version="2.18.1" />
   </ItemGroup>
 
   

Streetwriters.Common/Version.cs

@@ -0,0 +1,32 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Streetwriters.Common
+{
+    public class Version
+    {
+        public const int MAJOR = 2;
+        public const int MINOR = 3;
+        public const int PATCH = 0;
+        public static string AsString()
+        {
+            return $"{Version.MAJOR}.{Version.MINOR}.{Version.PATCH}";
+        }
+    }
+}

Streetwriters.Common/WampServers.cs

@@ -23,7 +23,6 @@ using System.Collections.Generic;
 using System.Reactive.Subjects;
 using System.Threading.Tasks;
 using Streetwriters.Common.Helpers;
-using Streetwriters.Common.Interfaces;
 using WampSharp.V2.Client;
 
 namespace Streetwriters.Common
@@ -37,28 +36,25 @@ namespace Streetwriters.Common
         public T Topics { get; set; } = new T();
         public string Realm { get; set; }
 
-        private async Task<IWampRealmProxy> GetChannelAsync(string topic)
-        {
-            if (!Channels.TryGetValue(topic, out IWampRealmProxy channel) || !channel.Monitor.IsConnected)
-            {
-                channel = await WampHelper.OpenWampChannelAsync(Address, Realm);
-                Channels.AddOrUpdate(topic, (key) => channel, (key, old) => channel);
-            }
-            return channel;
-        }
-
-        public async Task<V> GetServiceAsync<V>(string topic) where V : class
-        {
-            var channel = await GetChannelAsync(topic);
-            return channel.Services.GetCalleeProxy<V>();
-        }
-
         public async Task PublishMessageAsync<V>(string topic, V message)
         {
             try
             {
-                IWampRealmProxy channel = await GetChannelAsync(topic);
-                WampHelper.PublishMessage(channel, topic, message);
+                IWampRealmProxy channel;
+                if (Channels.ContainsKey(topic))
+                    channel = Channels[topic];
+                else
+                {
+                    channel = await WampHelper.OpenWampChannelAsync<V>(this.Address, this.Realm);
+                    Channels.TryAdd(topic, channel);
+                }
+                if (!channel.Monitor.IsConnected)
+                {
+                    Channels.TryRemove(topic, out IWampRealmProxy value);
+                    await PublishMessageAsync<V>(topic, message);
+                    return;
+                }
+                WampHelper.PublishMessage<V>(channel, topic, message);
             }
             catch (Exception ex)
             {
@@ -101,25 +97,23 @@ namespace Streetwriters.Common
 
     public class MessengerServerTopics
     {
-        public const string SendSSETopic = "co.streetwriters.sse.send";
+        public string SendSSETopic => "com.streetwriters.sse.send";
     }
 
     public class SubscriptionServerTopics
     {
-        public const string UserSubscriptionServiceTopic = "co.streetwriters.subscriptions.subscriptions";
-
-        public const string CreateSubscriptionTopic = "co.streetwriters.subscriptions.create";
-        public const string DeleteSubscriptionTopic = "co.streetwriters.subscriptions.delete";
+        public string CreateSubscriptionTopic => "com.streetwriters.subscriptions.create";
+        public string DeleteSubscriptionTopic => "com.streetwriters.subscriptions.delete";
     }
 
     public class IdentityServerTopics
     {
-        public const string UserAccountServiceTopic = "co.streetwriters.identity.users";
-        public const string ClearCacheTopic = "co.streetwriters.identity.clear_cache";
-        public const string DeleteUserTopic = "co.streetwriters.identity.delete_user";
+        public string CreateSubscriptionTopic => "com.streetwriters.subscriptions.create";
+        public string DeleteSubscriptionTopic => "com.streetwriters.subscriptions.delete";
     }
 
     public class NotesnookServerTopics
     {
+        public string DeleteUserTopic => "com.streetwriters.notesnook.user.delete";
     }
 }

Streetwriters.Data/Attributes/BsonCollection.cs

@@ -17,22 +17,20 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Collections.Generic;
-using System.Runtime.Serialization;
-using System.Text.Json.Serialization;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
+using System;
 
-namespace Streetwriters.Common.Messages
+namespace Streetwriters.Data.Attributes
 {
-    public class ClearCacheMessage
+    [AttributeUsage(AttributeTargets.Class, Inherited = false)]
+    public class BsonCollectionAttribute : Attribute
     {
-        public ClearCacheMessage(List<string> keys)
-        {
-            this.Keys = keys;
-        }
+        public string CollectionName { get; }
+        public string DatabaseName { get; }
 
-        [JsonPropertyName("keys")]
-        public List<string> Keys { get; set; }
+        public BsonCollectionAttribute(string databaseName, string collectionName)
+        {
+            CollectionName = collectionName;
+            DatabaseName = databaseName;
+        }
     }
 }

Streetwriters.Data/DbContexts/MongoDbContext.cs

@@ -17,44 +17,31 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using Microsoft.Extensions.Configuration;
+using MongoDB.Driver;
+using Streetwriters.Data.Interfaces;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.Extensions.Configuration;
-using MongoDB.Driver;
-using Streetwriters.Data.Interfaces;
 
 namespace Streetwriters.Data.DbContexts
 {
-    public class MongoDbContext(IMongoClient MongoClient) : IDbContext
+    public class MongoDbContext : IDbContext
     {
-        public static IMongoClient CreateMongoDbClient(IDbSettings dbSettings)
+        private IMongoDatabase Database { get; set; }
+        private MongoClient MongoClient { get; set; }
+        private readonly List<Func<IClientSessionHandle, CancellationToken, Task>> _commands;
+        private IDbSettings DbSettings { get; set; }
+        public MongoDbContext(IDbSettings dbSettings)
         {
-            var settings = MongoClientSettings.FromConnectionString(dbSettings.ConnectionString);
-            settings.MaxConnectionPoolSize = 500;
-            settings.MinConnectionPoolSize = 0;
-            settings.HeartbeatInterval = TimeSpan.FromSeconds(60);
-            return new MongoClient(settings);
+            DbSettings = dbSettings;
+            Configure();
+            // Every command will be stored and it'll be processed at SaveChanges
+            _commands = new List<Func<IClientSessionHandle, CancellationToken, Task>>();
         }
 
-        public static IMongoCollection<T> GetMongoCollection<T>(
-            IMongoClient client,
-            string databaseName,
-            string collectionName
-        )
-        {
-            return client
-                .GetDatabase(databaseName)
-                .GetCollection<T>(
-                    collectionName,
-                    new MongoCollectionSettings() { AssignIdOnInsert = true }
-                );
-        }
-
-        private readonly List<Func<IClientSessionHandle, CancellationToken, Task>> _commands = [];
-
         public async Task<int> SaveChanges()
         {
             try
@@ -63,17 +50,16 @@ namespace Streetwriters.Data.DbContexts
 
                 using (IClientSessionHandle session = await MongoClient.StartSessionAsync())
                 {
-#if (DEBUG || STAGING)
-                    await Parallel.ForEachAsync(_commands, async (c, ct) => await c(session, ct));
+#if DEBUG
+                    await Task.WhenAll(_commands.Select(c => c(session, default(CancellationToken))));
 #else
-                    await session.WithTransactionAsync(
-                        async (handle, token) =>
-                        {
-                            await Task.WhenAll(_commands.Select(c => c(handle, token)));
-                            return true;
-                        }
-                    );
+                    await session.WithTransactionAsync(async (handle, token) =>
+                    {
+                        await Task.WhenAll(_commands.Select(c => c(handle, token)));
+                        return true;
+                    });
 #endif
+
                 }
                 return count;
             }
@@ -85,6 +71,26 @@ namespace Streetwriters.Data.DbContexts
             }
         }
 
+        private void Configure()
+        {
+            if (MongoClient != null)
+            {
+                return;
+            }
+            var settings = MongoClientSettings.FromConnectionString(DbSettings.ConnectionString);
+            settings.MaxConnectionPoolSize = 5000;
+            settings.MinConnectionPoolSize = 300;
+            MongoClient = new MongoClient(settings);
+        }
+
+        public IMongoCollection<T> GetCollection<T>(string databaseName, string collectionName)
+        {
+            return MongoClient.GetDatabase(databaseName).GetCollection<T>(collectionName, new MongoCollectionSettings()
+            {
+                AssignIdOnInsert = true,
+            });
+        }
+
         public void AddCommand(Func<IClientSessionHandle, CancellationToken, Task> func)
         {
             _commands.Add(func);
@@ -94,5 +100,10 @@ namespace Streetwriters.Data.DbContexts
         {
             GC.SuppressFinalize(this);
         }
+
+        public Task DropDatabaseAsync()
+        {
+            return MongoClient.DropDatabaseAsync(DbSettings.DatabaseName);
+        }
     }
 }

Streetwriters.Data/Interfaces/IDbContext.cs

@@ -29,5 +29,6 @@ namespace Streetwriters.Data.Interfaces
     {
         void AddCommand(Func<IClientSessionHandle, CancellationToken, Task> func);
         Task<int> SaveChanges();
+        IMongoCollection<T> GetCollection<T>(string databaseName, string collectionName);
     }
 }

Streetwriters.Data/Repositories/Repository.cs

@@ -24,6 +24,7 @@ using System.Linq.Expressions;
 using System.Threading.Tasks;
 using MongoDB.Bson;
 using MongoDB.Driver;
+using Streetwriters.Data.Attributes;
 using Streetwriters.Data.Interfaces;
 
 namespace Streetwriters.Data.Repositories
@@ -31,14 +32,24 @@ namespace Streetwriters.Data.Repositories
     public class Repository<TEntity> where TEntity : class
     {
         protected readonly IDbContext dbContext;
-        public IMongoCollection<TEntity> Collection { get; set; }
+        protected IMongoCollection<TEntity> Collection { get; set; }
 
-        public Repository(IDbContext _dbContext, IMongoCollection<TEntity> collection)
+        public Repository(IDbContext _dbContext)
         {
             dbContext = _dbContext;
-            Collection = collection;
+            Collection = GetCollection();
         }
 
+        private protected IMongoCollection<TEntity> GetCollection()
+        {
+            var attribute = (BsonCollectionAttribute)typeof(TEntity).GetCustomAttributes(
+                    typeof(BsonCollectionAttribute),
+                    true).FirstOrDefault();
+            if (string.IsNullOrEmpty(attribute.CollectionName) || string.IsNullOrEmpty(attribute.DatabaseName)) throw new Exception("Could not get a valid collection or database name.");
+            return dbContext.GetCollection<TEntity>(attribute.DatabaseName, attribute.CollectionName);
+        }
+
+
         public virtual void Insert(TEntity obj)
         {
             dbContext.AddCommand((handle, ct) => Collection.InsertOneAsync(handle, obj, null, ct));

Streetwriters.Data/Streetwriters.Data.csproj

@@ -1,15 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
     
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.0" />
-    <PackageReference Include="MongoDB.Driver" Version="2.22.0" />
-    <PackageReference Include="MongoDB.Driver.Core" Version="2.22.0" />
-    <PackageReference Include="MongoDB.Bson" Version="2.22.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.13.2" />
+    <PackageReference Include="MongoDB.Driver.Core" Version="2.13.2" />
+    <PackageReference Include="MongoDB.Bson" Version="2.13.2" />
   </ItemGroup>
 
 </Project>

Streetwriters.Data/UnitOfWork.cs

@@ -23,16 +23,24 @@ using Streetwriters.Data.Interfaces;
 
 namespace Streetwriters.Data
 {
-    public class UnitOfWork(IDbContext dbContext) : IUnitOfWork
+    public class UnitOfWork : IUnitOfWork
     {
+        private readonly IDbContext dbContext;
+
+        public UnitOfWork(IDbContext _dbContext)
+        {
+            dbContext = _dbContext;
+        }
+
         public async Task<bool> Commit()
         {
-            return await dbContext.SaveChanges() > 0;
+            var changeAmount = await dbContext.SaveChanges();
+            return changeAmount > 0;
         }
 
         public void Dispose()
         {
-            dbContext.Dispose();
+            this.dbContext.Dispose();
         }
     }
 }

Streetwriters.Identity/Config.cs

@@ -20,7 +20,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using IdentityServer4;
 using IdentityServer4.Models;
 using Streetwriters.Common;
+using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace Streetwriters.Identity
 {
@@ -76,8 +78,8 @@ namespace Streetwriters.Identity
                     RefreshTokenUsage = TokenUsage.ReUse,
                     RefreshTokenExpiration = TokenExpiration.Sliding,
 
-                    AccessTokenLifetime = 6 * 3600, // 6 hours
-                    SlidingRefreshTokenLifetime = 45 * 3600 * 24, // 45 days
+                    AccessTokenLifetime = 3600, // 1 hour
+                    SlidingRefreshTokenLifetime = 15 * 60 * 60 * 24, // 15 days
                     AbsoluteRefreshTokenLifetime = 0, // 0 means infinite sliding lifetime
                     
                     // scopes that client has access to

Streetwriters.Identity/Controllers/AccountController.cs

@@ -21,26 +21,19 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
 using System.Security.Claims;
-using System.Text.Json;
 using System.Threading.Tasks;
 using AspNetCore.Identity.Mongo.Model;
-using IdentityServer4;
 using IdentityServer4.Configuration;
-using IdentityServer4.Models;
 using IdentityServer4.Stores;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.RateLimiting;
 using Streetwriters.Common;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Common.Models;
 using Streetwriters.Identity.Enums;
 using Streetwriters.Identity.Interfaces;
 using Streetwriters.Identity.Models;
-using Streetwriters.Identity.Services;
 using static IdentityServer4.IdentityServerConstants;
 
 namespace Streetwriters.Identity.Controllers
@@ -53,14 +46,14 @@ namespace Streetwriters.Identity.Controllers
     {
         private IPersistedGrantStore PersistedGrantStore { get; set; }
         private ITokenGenerationService TokenGenerationService { get; set; }
-        private IUserAccountService UserAccountService { get; set; }
-        public AccountController(UserManager<User> _userManager, ITemplatedEmailSender _emailSender,
+        private IUserClaimsPrincipalFactory<User> PrincipalFactory { get; set; }
+        private IdentityServerOptions ISOptions { get; set; }
+        public AccountController(UserManager<User> _userManager, IEmailSender _emailSender,
         SignInManager<User> _signInManager, RoleManager<MongoRole> _roleManager, IPersistedGrantStore store,
-        ITokenGenerationService tokenGenerationService, IMFAService _mfaService, IUserAccountService userAccountService) : base(_userManager, _emailSender, _signInManager, _roleManager, _mfaService)
+        ITokenGenerationService tokenGenerationService, IMFAService _mfaService) : base(_userManager, _emailSender, _signInManager, _roleManager, _mfaService)
         {
             PersistedGrantStore = store;
             TokenGenerationService = tokenGenerationService;
-            UserAccountService = userAccountService;
         }
 
         [HttpGet("confirm")]
@@ -72,7 +65,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.FindByIdAsync(userId);
-            if (!await UserService.IsUserValidAsync(UserManager, user, clientId)) return BadRequest($"Unable to find user with ID '{userId}'.");
+            if (!await IsUserValidAsync(user, clientId)) return BadRequest($"Unable to find user with ID '{userId}'.");
 
             switch (type)
             {
@@ -83,20 +76,30 @@ namespace Streetwriters.Identity.Controllers
                         var result = await UserManager.ConfirmEmailAsync(user, code);
                         if (!result.Succeeded) return BadRequest(result.Errors.ToErrors());
 
+
                         if (await UserManager.IsInRoleAsync(user, client.Id))
                         {
                             await client.OnEmailConfirmed(userId);
-                        }
-
-                        if (!await UserManager.GetTwoFactorEnabledAsync(user))
-                        {
-                            await MFAService.EnableMFAAsync(user, MFAMethods.Email);
-                            user = await UserManager.GetUserAsync(User);
+                            // if (client.WelcomeEmailTemplateId != null)
+                            //     await EmailSender.SendWelcomeEmailAsync(user.Email, client);
                         }
 
                         var redirectUrl = $"{client.EmailConfirmedRedirectURL}?userId={userId}";
                         return RedirectPermanent(redirectUrl);
                     }
+                // case TokenType.CHANGE_EMAIL:
+                //     {
+                //         var newEmail = user.Claims.Find((c) => c.ClaimType == "new_email");
+                //         if (newEmail == null) return BadRequest("Email change was not requested.");
+
+                //         var result = await UserManager.ChangeEmailAsync(user, newEmail.ClaimValue.ToString(), code);
+                //         if (result.Succeeded)
+                //         {
+                //             await UserManager.RemoveClaimAsync(user, newEmail.ToClaim());
+                //             return Ok("Email changed.");
+                //         }
+                //         return BadRequest("Could not change email.");
+                //     }
                 case TokenType.RESET_PASSWORD:
                     {
                         if (!await UserManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword", code))
@@ -113,19 +116,18 @@ namespace Streetwriters.Identity.Controllers
         }
 
         [HttpPost("verify")]
-        [EnableRateLimiting("strict")]
         public async Task<IActionResult> SendVerificationEmail([FromForm] string newEmail)
         {
             var client = Clients.FindClientById(User.FindFirstValue("client_id"));
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
 
             if (string.IsNullOrEmpty(newEmail))
             {
                 var code = await UserManager.GenerateEmailConfirmationTokenAsync(user);
-                var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.CONFRIM_EMAIL);
+                var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.CONFRIM_EMAIL, Request.Scheme);
                 await EmailSender.SendConfirmationEmailAsync(user.Email, callbackUrl, client);
             }
             else
@@ -136,29 +138,66 @@ namespace Streetwriters.Identity.Controllers
             return Ok();
         }
 
+        [HttpPost("unregister")]
+        public async Task<IActionResult> UnregisterAccountAync([FromForm] DeleteAccountForm form)
+        {
+            var client = Clients.FindClientById(User.FindFirstValue("client_id"));
+            if (client == null) return BadRequest("Invalid client_id.");
+
+            var user = await UserManager.GetUserAsync(User);
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+
+            if (!await UserManager.CheckPasswordAsync(user, form.Password))
+            {
+                return Unauthorized();
+            }
+
+            await UserManager.RemoveFromRoleAsync(user, client.Id);
+
+            IdentityUserClaim<string> statusClaim = user.Claims.FirstOrDefault((c) => c.ClaimType == $"{client.Id}:status");
+            await UserManager.RemoveClaimAsync(user, statusClaim.ToClaim());
+            return Ok();
+        }
+
         [HttpGet]
         public async Task<IActionResult> GetUserAccount()
         {
             var client = Clients.FindClientById(User.FindFirstValue("client_id"));
             if (client == null) return BadRequest("Invalid client_id.");
+
             var user = await UserManager.GetUserAsync(User);
-            return Ok(UserAccountService.GetUserAsync(client.Id, user.Id.ToString()));
+            if (!await IsUserValidAsync(user, client.Id))
+                return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+
+            return Ok(new UserModel
+            {
+                UserId = user.Id.ToString(),
+                Email = user.Email,
+                IsEmailConfirmed = user.EmailConfirmed,
+                // PhoneNumber = user.PhoneNumberConfirmed ? user.PhoneNumber : null,
+                MFA = new MFAConfig
+                {
+                    IsEnabled = user.TwoFactorEnabled,
+                    PrimaryMethod = MFAService.GetPrimaryMethod(user),
+                    SecondaryMethod = MFAService.GetSecondaryMethod(user),
+                    RemainingValidCodes = await MFAService.GetRemainingValidCodesAsync(user)
+                }
+            });
         }
 
         [HttpPost("recover")]
         [AllowAnonymous]
-        [EnableRateLimiting("strict")]
         public async Task<IActionResult> ResetUserPassword([FromForm] ResetPasswordForm form)
         {
             var client = Clients.FindClientById(form.ClientId);
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.FindByEmailAsync(form.Email);
-            if (!await UserService.IsUserValidAsync(UserManager, user, form.ClientId)) return Ok();
+            if (!await IsUserValidAsync(user, form.ClientId)) return Ok();
 
             var code = await UserManager.GenerateUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword");
-            var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.RESET_PASSWORD);
-#if (DEBUG || STAGING)
+            var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.RESET_PASSWORD, Request.Scheme);
+#if DEBUG
             return Ok(callbackUrl);
 #else
             await Slogger<AccountController>.Info("ResetUserPassword", user.Email, callbackUrl);
@@ -174,7 +213,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
 
             var subjectId = User.FindFirstValue("sub");
             var jti = User.FindFirstValue("jti");
@@ -201,7 +240,7 @@ namespace Streetwriters.Identity.Controllers
         {
             if (!Clients.IsValidClient(form.ClientId)) return BadRequest("Invalid clientId.");
             var user = await UserManager.FindByIdAsync(form.UserId);
-            if (!await UserService.IsUserValidAsync(UserManager, user, form.ClientId))
+            if (!await IsUserValidAsync(user, form.ClientId))
                 return BadRequest($"Unable to find user with ID '{form.UserId}'.");
 
             if (!await UserManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, "PasswordResetAuthorizationCode", form.Code))
@@ -210,7 +249,6 @@ namespace Streetwriters.Identity.Controllers
             return Ok(new
             {
                 access_token = token,
-                scope = string.Join(' ', Config.ApiScopes.Select(s => s.Name)),
                 expires_in = 18000
             });
         }
@@ -222,7 +260,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id))
+            if (!await IsUserValidAsync(user, client.Id))
                 return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
 
             switch (form.Type)
@@ -239,7 +277,7 @@ namespace Streetwriters.Identity.Controllers
                                 if (result.Succeeded)
                                 {
                                     await UserManager.SetUserNameAsync(user, form.NewEmail);
-                                    await SendLogoutMessageAsync(user.Id.ToString(), "Email changed.");
+                                    await SendEmailChangedMessageAsync(user.Id.ToString());
                                     return Ok();
                                 }
                             }
@@ -251,7 +289,7 @@ namespace Streetwriters.Identity.Controllers
                         var result = await UserManager.ChangePasswordAsync(user, form.OldPassword, form.NewPassword);
                         if (result.Succeeded)
                         {
-                            await SendLogoutMessageAsync(user.Id.ToString(), "Password changed.");
+                            await SendPasswordChangedMessageAsync(user.Id.ToString());
                             return Ok();
                         }
                         return BadRequest(result.Errors.ToErrors());
@@ -261,27 +299,15 @@ namespace Streetwriters.Identity.Controllers
                         var result = await UserManager.RemovePasswordAsync(user);
                         if (result.Succeeded)
                         {
-                            await MFAService.ResetMFAAsync(user);
                             result = await UserManager.AddPasswordAsync(user, form.NewPassword);
                             if (result.Succeeded)
                             {
-                                await SendLogoutMessageAsync(user.Id.ToString(), "Password reset.");
+                                await SendPasswordChangedMessageAsync(user.Id.ToString());
                                 return Ok();
                             }
                         }
                         return BadRequest(result.Errors.ToErrors());
                     }
-                case "change_marketing_consent":
-                    {
-                        var claimType = $"{client.Id}:marketing_consent";
-                        var claims = await UserManager.GetClaimsAsync(user);
-                        var marketingConsentClaim = claims.FirstOrDefault((claim) => claim.Type == claimType);
-                        if (marketingConsentClaim != null) await UserManager.RemoveClaimAsync(user, marketingConsentClaim);
-                        if (!form.Enabled)
-                            await UserManager.AddClaimAsync(user, new Claim(claimType, "false"));
-                        return Ok();
-                    }
-
             }
             return BadRequest("Invalid type.");
         }
@@ -293,7 +319,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id)) return BadRequest($"Unable to find user with ID '{user.Id}'.");
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{user.Id.ToString()}'.");
 
             var jti = User.FindFirstValue("jti");
 
@@ -302,44 +328,43 @@ namespace Streetwriters.Identity.Controllers
                 ClientId = client.Id,
                 SubjectId = user.Id.ToString()
             });
-            var refreshTokenKey = GetHashedKey(refresh_token, PersistedGrantTypes.RefreshToken);
-            var removedKeys = new List<string>();
             foreach (var grant in grants)
             {
-                if (!all && (grant.Data.Contains(jti) || grant.Key == refreshTokenKey)) continue;
+                if (!all && (grant.Data.Contains(jti) || grant.Data.Contains(refresh_token))) continue;
                 await PersistedGrantStore.RemoveAsync(grant.Key);
-                removedKeys.Add(grant.Key);
             }
-
-            await WampServers.NotesnookServer.PublishMessageAsync(IdentityServerTopics.ClearCacheTopic, new ClearCacheMessage(removedKeys));
-            await WampServers.MessengerServer.PublishMessageAsync(IdentityServerTopics.ClearCacheTopic, new ClearCacheMessage(removedKeys));
-            await WampServers.SubscriptionServer.PublishMessageAsync(IdentityServerTopics.ClearCacheTopic, new ClearCacheMessage(removedKeys));
-            await SendLogoutMessageAsync(user.Id.ToString(), "Session revoked.");
             return Ok();
         }
 
-        private static string GetHashedKey(string value, string grantType)
+        private async Task SendPasswordChangedMessageAsync(string userId)
         {
-            return (value + ":" + grantType).Sha256();
-        }
-
-        private async Task SendLogoutMessageAsync(string userId, string reason)
-        {
-            await SendMessageAsync(userId, new Message
-            {
-                Type = "logout",
-                Data = JsonSerializer.Serialize(new { reason })
-            });
-        }
-
-        private async Task SendMessageAsync(string userId, Message message)
-        {
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+            await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
             {
                 UserId = userId,
                 OriginTokenId = User.FindFirstValue("jti"),
-                Message = message
+                Message = new Message
+                {
+                    Type = "userPasswordChanged"
+                }
             });
         }
+
+        private async Task SendEmailChangedMessageAsync(string userId)
+        {
+            await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
+            {
+                UserId = userId,
+                OriginTokenId = User.FindFirstValue("jti"),
+                Message = new Message
+                {
+                    Type = "userEmailChanged"
+                }
+            });
+        }
+
+        public async Task<bool> IsUserValidAsync(User user, string clientId)
+        {
+            return user != null && await UserManager.IsInRoleAsync(user, clientId);
+        }
     }
 }

Streetwriters.Identity/Controllers/IdentityControllerBase.cs

@@ -36,12 +36,12 @@ namespace Streetwriters.Identity.Controllers
         protected UserManager<User> UserManager { get; set; }
         protected SignInManager<User> SignInManager { get; set; }
         protected RoleManager<MongoRole> RoleManager { get; set; }
-        protected ITemplatedEmailSender EmailSender { get; set; }
+        protected IEmailSender EmailSender { get; set; }
         protected UrlEncoder UrlEncoder { get; set; }
         protected IMFAService MFAService { get; set; }
         public IdentityControllerBase(
             UserManager<User> _userManager,
-            ITemplatedEmailSender _emailSender,
+            IEmailSender _emailSender,
             SignInManager<User> _signInManager,
             RoleManager<MongoRole> _roleManager,
             IMFAService _mfaService

Streetwriters.Identity/Controllers/MFAController.cs

@@ -26,7 +26,6 @@ using AspNetCore.Identity.Mongo.Model;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.RateLimiting;
 using Streetwriters.Common;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Models;
@@ -42,7 +41,7 @@ namespace Streetwriters.Identity.Controllers
     [Authorize(LocalApi.PolicyName)]
     public class MFAController : IdentityControllerBase
     {
-        public MFAController(UserManager<User> _userManager, ITemplatedEmailSender _emailSender,
+        public MFAController(UserManager<User> _userManager, IEmailSender _emailSender,
         SignInManager<User> _signInManager, RoleManager<MongoRole> _roleManager, IMFAService _mfaService) : base(_userManager, _emailSender, _signInManager, _roleManager, _mfaService) { }
 
         [HttpPost]
@@ -75,9 +74,21 @@ namespace Streetwriters.Identity.Controllers
         }
 
         [HttpDelete]
-        public IActionResult Disable2FA()
+        public async Task<IActionResult> Disable2FA()
         {
-            return BadRequest("2FA is mandatory and cannot be disabled.");
+            var user = await UserManager.GetUserAsync(User);
+
+            if (!await UserManager.GetTwoFactorEnabledAsync(user))
+            {
+                return BadRequest("Cannot disable 2FA as it's not currently enabled");
+            }
+
+            if (await MFAService.DisableMFAAsync(user))
+            {
+                return Ok();
+            }
+
+            return BadRequest("Failed to disable 2FA.");
         }
 
         [HttpGet("codes")]
@@ -91,7 +102,6 @@ namespace Streetwriters.Identity.Controllers
         [HttpPost("send")]
         [Authorize("mfa")]
         [Authorize(LocalApi.PolicyName)]
-        [EnableRateLimiting("strict")]
         public async Task<IActionResult> RequestCode([FromForm] string type)
         {
             var client = Clients.FindClientById(User.FindFirstValue("client_id"));

Streetwriters.Identity/Controllers/SignupController.cs

@@ -17,6 +17,7 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
@@ -25,9 +26,7 @@ using AspNetCore.Identity.Mongo.Model;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.RateLimiting;
 using Streetwriters.Common;
-using Streetwriters.Common.Enums;
 using Streetwriters.Common.Models;
 using Streetwriters.Identity.Enums;
 using Streetwriters.Identity.Interfaces;
@@ -40,7 +39,7 @@ namespace Streetwriters.Identity.Controllers
     [Route("signup")]
     public class SignupController : IdentityControllerBase
     {
-        public SignupController(UserManager<User> _userManager, ITemplatedEmailSender _emailSender,
+        public SignupController(UserManager<User> _userManager, IEmailSender _emailSender,
         SignInManager<User> _signInManager, RoleManager<MongoRole> _roleManager, IMFAService _mfaService) : base(_userManager, _emailSender, _signInManager, _roleManager, _mfaService)
         { }
 
@@ -52,90 +51,70 @@ namespace Streetwriters.Identity.Controllers
 
         [HttpPost]
         [AllowAnonymous]
-        [EnableRateLimiting("strict")]
         public async Task<IActionResult> Signup([FromForm] SignupForm form)
         {
-            if (Constants.DISABLE_SIGNUPS)
-                return BadRequest(new string[] { "Creating new accounts is not allowed." });
-            try
+            var client = Clients.FindClientById(form.ClientId);
+            if (client == null) return BadRequest(new string[] { "Invalid client id." });
+
+            await AddClientRoleAsync(client.Id);
+
+            // email addresses must be case-insensitive
+            form.Email = form.Email.ToLowerInvariant();
+            form.Username = form.Username?.ToLowerInvariant();
+
+            if (!await EmailAddressValidator.IsEmailAddressValidAsync(form.Email)) return BadRequest(new string[] { "Invalid email address." });
+
+            var result = await UserManager.CreateAsync(new User
             {
-                var client = Clients.FindClientById(form.ClientId);
-                if (client == null) return BadRequest(new string[] { "Invalid client id." });
+                Email = form.Email,
+                EmailConfirmed = false,
+                UserName = form.Username ?? form.Email,
+            }, form.Password);
 
-                await AddClientRoleAsync(client.Id);
+            if (result.Errors.Any((e) => e.Code == "DuplicateEmail"))
+            {
+                var user = await UserManager.FindByEmailAsync(form.Email);
 
-                // email addresses must be case-insensitive
-                form.Email = form.Email.ToLowerInvariant();
-                form.Username = form.Username?.ToLowerInvariant();
-
-                if (!await EmailAddressValidator.IsEmailAddressValidAsync(form.Email)) return BadRequest(new string[] { "Invalid email address." });
-
-                var result = await UserManager.CreateAsync(new User
+                if (!await UserManager.IsInRoleAsync(user, client.Id))
                 {
-                    Email = form.Email,
-                    EmailConfirmed = Constants.IS_SELF_HOSTED,
-                    UserName = form.Username ?? form.Email,
-                }, form.Password);
-
-                if (result.Errors.Any((e) => e.Code == "DuplicateEmail"))
-                {
-                    var user = await UserManager.FindByEmailAsync(form.Email);
-
-                    if (!await UserManager.IsInRoleAsync(user, client.Id))
+                    if (!await UserManager.CheckPasswordAsync(user, form.Password))
                     {
-                        if (!await UserManager.CheckPasswordAsync(user, form.Password))
-                        {
-                            // TODO
-                            await UserManager.RemovePasswordAsync(user);
-                            await UserManager.AddPasswordAsync(user, form.Password);
-                        }
-                        await MFAService.DisableMFAAsync(user);
-                        await UserManager.AddToRoleAsync(user, client.Id);
+                        // TODO
+                        await UserManager.RemovePasswordAsync(user);
+                        await UserManager.AddPasswordAsync(user, form.Password);
                     }
-                    else
-                    {
-                        return BadRequest(new string[] { "Invalid email address.." });
-                    }
-
-                    return Ok(new
-                    {
-                        userId = user.Id.ToString()
-                    });
-                }
-
-                if (result.Succeeded)
-                {
-                    var user = await UserManager.FindByEmailAsync(form.Email);
                     await UserManager.AddToRoleAsync(user, client.Id);
-                    if (Constants.IS_SELF_HOSTED)
-                    {
-                        await UserManager.AddClaimAsync(user, UserService.SubscriptionTypeToClaim(client.Id, Common.Enums.SubscriptionType.PREMIUM));
-                    }
-                    else
-                    {
-                        await UserManager.AddClaimAsync(user, new Claim("platform", PlatformFromUserAgent(base.HttpContext.Request.Headers.UserAgent)));
-                        var code = await UserManager.GenerateEmailConfirmationTokenAsync(user);
-                        var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.CONFRIM_EMAIL);
-                        await EmailSender.SendConfirmationEmailAsync(user.Email, callbackUrl, client);
-                    }
-                    return Ok(new
-                    {
-                        userId = user.Id.ToString()
-                    });
+                }
+                else
+                {
+                    return BadRequest(new string[] { "Invalid email address." });
                 }
 
-                return BadRequest(result.Errors.ToErrors());
+                return Ok(new
+                {
+                    userId = user.Id.ToString()
+                });
             }
-            catch (System.Exception ex)
-            {
-                await Slogger<SignupController>.Error("Signup", ex.ToString());
-                return BadRequest("Failed to create an account.");
-            }
-        }
 
-        string PlatformFromUserAgent(string userAgent)
-        {
-            return userAgent.Contains("okhttp/") ? "android" : userAgent.Contains("Darwin/") || userAgent.Contains("CFNetwork/") ? "ios" : "web";
+            if (result.Succeeded)
+            {
+                var user = await UserManager.FindByEmailAsync(form.Email);
+
+                await UserManager.AddToRoleAsync(user, client.Id);
+                if (Constants.IS_SELF_HOSTED)
+                    await UserManager.AddClaimAsync(user, UserService.SubscriptionTypeToClaim(client.Id, Common.Enums.SubscriptionType.PREMIUM));
+
+                var code = await UserManager.GenerateEmailConfirmationTokenAsync(user);
+                var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.CONFRIM_EMAIL, Request.Scheme);
+                await EmailSender.SendConfirmationEmailAsync(user.Email, callbackUrl, client);
+
+                return Ok(new
+                {
+                    userId = user.Id.ToString()
+                });
+            }
+
+            return BadRequest(result.Errors.ToErrors());
         }
     }
 }

Streetwriters.Identity/Dockerfile

@@ -1,50 +1,28 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build
 WORKDIR /app
 
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
-ARG TARGETARCH
-ARG BUILDPLATFORM
-ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
-
-WORKDIR /src
-
+# restore all project dependencies
 COPY Streetwriters.Data/*.csproj ./Streetwriters.Data/
+RUN dotnet restore /app/Streetwriters.Data/Streetwriters.Data.csproj --use-current-runtime
+
 COPY Streetwriters.Common/*.csproj ./Streetwriters.Common/
+RUN dotnet restore /app/Streetwriters.Common/Streetwriters.Common.csproj --use-current-runtime
+
 COPY Streetwriters.Identity/*.csproj ./Streetwriters.Identity/
+RUN dotnet restore /app/Streetwriters.Identity/Streetwriters.Identity.csproj --use-current-runtime
 
-# restore dependencies
-RUN dotnet restore -v d /src/Streetwriters.Identity/Streetwriters.Identity.csproj --use-current-runtime
-
+# copy everything else
 COPY Streetwriters.Data/ ./Streetwriters.Data/
 COPY Streetwriters.Common/ ./Streetwriters.Common/
 COPY Streetwriters.Identity/ ./Streetwriters.Identity/
 
-WORKDIR /src/Streetwriters.Identity/
+# build
+WORKDIR /app/Streetwriters.Identity/
+ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
+RUN dotnet publish -c Release -o /app/out --use-current-runtime --self-contained false --no-restore
 
-RUN dotnet build -c Release -o /app/build -a $TARGETARCH
-
-FROM build AS publish
-RUN dotnet publish -c Release -o /app/publish \
-  #--runtime alpine-x64 \
-  --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
-  /p:PublishSingleFile=true \
-  /p:JsonSerializerIsReflectionEnabledByDefault=true \
-  -a $TARGETARCH
-
-FROM --platform=$BUILDPLATFORM base AS final
-ARG TARGETARCH
-ARG BUILDPLATFORM
-
-# create a new user and change directory ownership
-RUN adduser --disabled-password \
-  --home /app \
-  --gecos '' dotnetuser && chown -R dotnetuser /app
-
-# impersonate into the new user
-USER dotnetuser
+# final stage/image
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
 WORKDIR /app
-
-COPY --from=publish /app/publish .
-ENTRYPOINT ["./Streetwriters.Identity"]
+COPY --from=build /app/out .
+ENTRYPOINT ["dotnet", "Streetwriters.Identity.dll"]

Streetwriters.Identity/Extensions/UrlExtensions.cs

@@ -29,20 +29,19 @@ namespace Microsoft.AspNetCore.Mvc
 {
     public static class UrlHelperExtensions
     {
-        public static string TokenLink(this IUrlHelper urlHelper, string userId, string code, string clientId, TokenType type)
+        public static string TokenLink(this IUrlHelper urlHelper, string userId, string code, string clientId, TokenType type, string scheme)
         {
 
             return urlHelper.ActionLink(
-#if (DEBUG || STAGING)
+#if DEBUG
             host: $"{Servers.IdentityServer.Hostname}:{Servers.IdentityServer.Port}",
-            protocol: "http",
 #else
-            host: Servers.IdentityServer.PublicURL.Host,
-            protocol: Servers.IdentityServer.PublicURL.Scheme,
+            host: Servers.IdentityServer.Domain,  
 #endif
             action: nameof(AccountController.ConfirmToken),
             controller: "Account",
-            values: new { userId, code, clientId, type });
+            values: new { userId, code, clientId, type },
+            protocol: scheme);
 
         }
     }

Streetwriters.Identity/Helpers/PasswordHelper.cs

@@ -19,7 +19,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Text;
-using Geralt;
+using Sodium;
 
 namespace Streetwriters.Identity.Helpers
 {
@@ -27,14 +27,12 @@ namespace Streetwriters.Identity.Helpers
     {
         public static bool VerifyPassword(string password, string hash)
         {
-            return Argon2id.VerifyHash(Encoding.UTF8.GetBytes(hash), Encoding.UTF8.GetBytes(password));
+            return PasswordHash.ArgonHashStringVerify(hash, password);
         }
 
         public static string CreatePasswordHash(string password)
         {
-            Span<byte> hash = new(new byte[128]);
-            Argon2id.ComputeHash(hash, Encoding.UTF8.GetBytes(password), 3, 65536);
-            return Encoding.UTF8.GetString(hash);
+            return PasswordHash.ArgonHashString(password, 3, 65536);
         }
     }
 }

Streetwriters.Identity/Interfaces/IEmailSender.cs

@@ -22,7 +22,7 @@ using Streetwriters.Common.Interfaces;
 
 namespace Streetwriters.Identity.Interfaces
 {
-    public interface ITemplatedEmailSender
+    public interface IEmailSender
     {
         Task SendConfirmationEmailAsync(string email, string callbackUrl, IClient client);
         Task SendChangeEmailConfirmationAsync(string email, string code, IClient client);

Streetwriters.Identity/Interfaces/IMFAService.cs

@@ -28,13 +28,11 @@ namespace Streetwriters.Identity.Interfaces
     {
         Task EnableMFAAsync(User user, string primaryMethod);
         Task<bool> DisableMFAAsync(User user);
-        Task<bool> ResetMFAAsync(User user);
         Task SetSecondaryMethodAsync(User user, string secondaryMethod);
         string GetPrimaryMethod(User user);
         string GetSecondaryMethod(User user);
         Task<int> GetRemainingValidCodesAsync(User user);
         bool IsValidMFAMethod(string method);
-        bool IsValidMFAMethod(string method, User user);
         Task<AuthenticatorDetails> GetAuthenticatorDetailsAsync(User user, IClient client);
         Task SendOTPAsync(User user, IClient client, MultiFactorSetupForm form, bool isSetup = false);
         Task<bool> VerifyOTPAsync(User user, string code, string method);

Streetwriters.Identity/Interfaces/ISMSSender.cs

@@ -24,7 +24,7 @@ namespace Streetwriters.Identity.Interfaces
 {
     public interface ISMSSender
     {
-        Task<string> SendOTPAsync(string number, IClient client);
-        Task<bool> VerifyOTPAsync(string id, string code);
+        string SendOTP(string number, IClient client);
+        bool VerifyOTP(string id, string code);
     }
 }

Streetwriters.Identity/MessageHandlers/CreateSubscription.cs

@@ -36,7 +36,7 @@ namespace Streetwriters.Identity.MessageHandlers
             var client = Clients.FindClientByAppId(message.AppId);
             if (client == null || user == null) return;
 
-            IdentityUserClaim<string> statusClaim = user.Claims.FirstOrDefault((c) => c.ClaimType == UserService.GetClaimKey(client.Id));
+            IdentityUserClaim<string> statusClaim = user.Claims.FirstOrDefault((c) => c.ClaimType == $"{client.Id}:status");
             Claim subscriptionClaim = UserService.SubscriptionTypeToClaim(client.Id, message.Type);
             if (statusClaim?.ClaimValue == subscriptionClaim.Value) return;
             if (statusClaim != null)

Streetwriters.Identity/Models/DeleteAccountForm.cs

@@ -17,11 +17,17 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-namespace Notesnook.API.Models
+using System.ComponentModel.DataAnnotations;
+using System.Runtime.Serialization;
+
+namespace Streetwriters.Identity.Models
 {
-  public class UserKeys
-  {
-    public EncryptedData AttachmentsKey { get; set; }
-    public EncryptedData MonographPasswordsKey { get; set; }
-  }
+    public class DeleteAccountForm
+    {
+        [Required]
+        public string Password
+        {
+            get; set;
+        }
+    }
 }

Streetwriters.Identity/Models/EmailTemplate.cs

@@ -0,0 +1,33 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using Streetwriters.Identity.Interfaces;
+
+namespace Streetwriters.Identity.Models
+{
+    public class EmailTemplate : IEmailTemplate
+    {
+        public int? Id { get; set; }
+        public object Data { get; set; }
+        public long? SendAt { get; set; }
+        public string Subject { get; set; }
+        public string Html { get; set; }
+        public string Text { get; set; }
+    }
+}

Streetwriters.Identity/Models/UpdateUserForm.cs

@@ -32,12 +32,6 @@ namespace Streetwriters.Identity.Models
             get; set;
         }
 
-        [BindProperty(Name = "enabled")]
-        public bool Enabled
-        {
-            get; set;
-        }
-
         [BindProperty(Name = "old_password")]
         public string OldPassword
         {

Streetwriters.Identity/Program.cs

@@ -33,7 +33,7 @@ namespace Streetwriters.Identity
     {
         public static async Task Main(string[] args)
         {
-#if (DEBUG || STAGING)
+#if DEBUG
             DotNetEnv.Env.TraversePath().Load(".env.local");
 #else
             DotNetEnv.Env.TraversePath().Load(".env");

Streetwriters.Identity/Services/CustomIntrospectionResponseGenerator.cs

@@ -46,7 +46,6 @@ namespace Streetwriters.Identity.Services
             if (result.TryGetValue("sub", out object userId))
             {
                 var user = await UserManager.FindByIdAsync(userId.ToString());
-                if (user == null || user.Claims == null) return result;
 
                 var verifiedClaim = user.Claims.Find((c) => c.ClaimType == "verified");
                 if (verifiedClaim != null)

Streetwriters.Identity/Services/CustomPersistedGrantDbContext.cs

@@ -67,7 +67,7 @@ namespace Streetwriters.Identity.Services
 
         public Task RemoveExpired()
         {
-            return Remove(x => x.Type == "reference_token" && x.Expiration.HasValue && x.Expiration.Value < DateTime.UtcNow);
+            return Remove(x => x.Expiration < DateTime.UtcNow.AddHours(12));
         }
 
         public Task InsertOrUpdate(Expression<Func<PersistedGrant, bool>> filter, PersistedGrant entity)

Streetwriters.Identity/Services/EmailAddressValidator.cs

@@ -3,14 +3,13 @@ using System.Collections.Generic;
 using System.Net.Http;
 using System.Threading.Tasks;
 using Streetwriters.Common;
-using System.Linq;
 
 namespace Streetwriters.Identity.Services
 {
     public class EmailAddressValidator
     {
         private static DateTimeOffset LAST_FETCH_TIME = DateTimeOffset.MinValue;
-        private static HashSet<string> BLACKLISTED_DOMAINS = new();
+        private static HashSet<string> BLACKLISTED_DOMAINS = new HashSet<string>();
 
         public static async Task<bool> IsEmailAddressValidAsync(string email)
         {
@@ -20,9 +19,8 @@ namespace Streetwriters.Identity.Services
                 if (LAST_FETCH_TIME.AddDays(1) < DateTimeOffset.UtcNow)
                 {
                     var httpClient = new HttpClient();
-                    var domainsList = await httpClient.GetStringAsync("https://raw.githubusercontent.com/disposable-email-domains/disposable-email-domains/master/disposable_email_blocklist.conf");
-                    var domains = domainsList.Split('\n').Where(line => !string.IsNullOrWhiteSpace(line) && !line.TrimStart().StartsWith("//"));
-                    BLACKLISTED_DOMAINS = new HashSet<string>(domains, StringComparer.OrdinalIgnoreCase);
+                    var domainsList = await httpClient.GetStringAsync("https://disposable.github.io/disposable-email-domains/domains.txt");
+                    BLACKLISTED_DOMAINS = new HashSet<string>(domainsList.Split('\n'));
                     LAST_FETCH_TIME = DateTimeOffset.UtcNow;
                 }
 

Streetwriters.Identity/Services/EmailSender.cs

@@ -0,0 +1,275 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Threading.Tasks;
+using Streetwriters.Identity.Interfaces;
+using SendGrid;
+using SendGrid.Helpers.Mail;
+using Streetwriters.Common;
+using Streetwriters.Common.Interfaces;
+using Streetwriters.Identity.Models;
+using System;
+using System.Net.Http;
+using System.Text.Json;
+using System.Net.Http.Headers;
+using System.Text.Json.Serialization;
+using MailKit.Net.Smtp;
+using MailKit;
+using MimeKit;
+using System.IO;
+using Scriban;
+using WebMarkupMin.Core;
+using WebMarkupMin.Core.Loggers;
+using MimeKit.Cryptography;
+using Org.BouncyCastle.Bcpg.OpenPgp;
+using System.Linq;
+using System.Threading;
+using Org.BouncyCastle.Bcpg;
+using System.Text;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Options;
+
+namespace Streetwriters.Identity.Services
+{
+    public class EmailSender : IEmailSender, IAsyncDisposable
+    {
+        NNGnuPGContext NNGnuPGContext { get; set; }
+        SmtpClient mailClient;
+        public EmailSender(IConfiguration configuration)
+        {
+            NNGnuPGContext = new NNGnuPGContext(configuration.GetSection("PgpKeySettings"));
+            mailClient = new SmtpClient();
+        }
+
+        EmailTemplate Email2FATemplate = new EmailTemplate
+        {
+            Html = ReadMinifiedHtmlFile("Templates/Email2FACode.html"),
+            Text = File.ReadAllText("Templates/Email2FACode.txt"),
+            Subject = "Your {{app_name}} account 2FA code",
+        };
+
+        EmailTemplate ConfirmEmailTemplate = new EmailTemplate
+        {
+            Html = ReadMinifiedHtmlFile("Templates/ConfirmEmail.html"),
+            Text = File.ReadAllText("Templates/ConfirmEmail.txt"),
+            Subject = "Confirm your {{app_name}} account",
+        };
+
+        EmailTemplate ConfirmChangeEmailTemplate = new EmailTemplate
+        {
+            Html = ReadMinifiedHtmlFile("Templates/EmailChangeConfirmation.html"),
+            Text = File.ReadAllText("Templates/EmailChangeConfirmation.txt"),
+            Subject = "Change {{app_name}} account email address",
+        };
+
+        EmailTemplate PasswordResetEmailTemplate = new EmailTemplate
+        {
+            Html = ReadMinifiedHtmlFile("Templates/ResetAccountPassword.html"),
+            Text = File.ReadAllText("Templates/ResetAccountPassword.txt"),
+            Subject = "Reset {{app_name}} account password",
+        };
+
+        EmailTemplate FailedLoginAlertTemplate = new EmailTemplate
+        {
+            Html = ReadMinifiedHtmlFile("Templates/FailedLoginAlert.html"),
+            Text = File.ReadAllText("Templates/FailedLoginAlert.txt"),
+            Subject = "Failed login attempt on your {{app_name}} account",
+        };
+
+        public async Task Send2FACodeEmailAsync(string email, string code, IClient client)
+        {
+            var template = new EmailTemplate
+            {
+                Html = Email2FATemplate.Html,
+                Text = Email2FATemplate.Text,
+                Subject = Email2FATemplate.Subject,
+                Data = new
+                {
+                    app_name = client.Name,
+                    code = code
+                }
+            };
+            await SendEmailAsync(email, template, client);
+        }
+
+        public async Task SendConfirmationEmailAsync(string email, string callbackUrl, IClient client)
+        {
+            var template = new EmailTemplate
+            {
+                Html = ConfirmEmailTemplate.Html,
+                Text = ConfirmEmailTemplate.Text,
+                Subject = ConfirmEmailTemplate.Subject,
+                Data = new
+                {
+                    app_name = client.Name,
+                    confirm_link = callbackUrl
+                }
+            };
+            await SendEmailAsync(email, template, client);
+        }
+
+        public async Task SendChangeEmailConfirmationAsync(string email, string code, IClient client)
+        {
+            var template = new EmailTemplate
+            {
+                Html = ConfirmChangeEmailTemplate.Html,
+                Text = ConfirmChangeEmailTemplate.Text,
+                Subject = ConfirmChangeEmailTemplate.Subject,
+                Data = new
+                {
+                    app_name = client.Name,
+                    code = code
+                }
+            };
+            await SendEmailAsync(email, template, client);
+        }
+
+        public async Task SendPasswordResetEmailAsync(string email, string callbackUrl, IClient client)
+        {
+            var template = new EmailTemplate
+            {
+                Html = PasswordResetEmailTemplate.Html,
+                Text = PasswordResetEmailTemplate.Text,
+                Subject = PasswordResetEmailTemplate.Subject,
+                Data = new
+                {
+                    app_name = client.Name,
+                    reset_link = callbackUrl
+                }
+            };
+            await SendEmailAsync(email, template, client);
+        }
+
+
+        public async Task SendFailedLoginAlertAsync(string email, string deviceInfo, IClient client)
+        {
+            var template = new EmailTemplate
+            {
+                Html = FailedLoginAlertTemplate.Html,
+                Text = FailedLoginAlertTemplate.Text,
+                Subject = FailedLoginAlertTemplate.Subject,
+                Data = new
+                {
+                    app_name = client.Name,
+                    device_info = deviceInfo.Replace("\n", "<br>")
+                }
+            };
+            await SendEmailAsync(email, template, client);
+        }
+
+        private async Task SendEmailAsync(string email, IEmailTemplate template, IClient client)
+        {
+            if (!mailClient.IsConnected)
+            {
+                if (int.TryParse(Constants.SMTP_PORT, out int port))
+                {
+                    await mailClient.ConnectAsync(Constants.SMTP_HOST, port, MailKit.Security.SecureSocketOptions.StartTls);
+                }
+                else
+                {
+                    throw new InvalidDataException("SMTP_PORT is not a valid integer value.");
+                }
+            }
+
+            if (!mailClient.IsAuthenticated)
+                await mailClient.AuthenticateAsync(Constants.SMTP_USERNAME, Constants.SMTP_PASSWORD);
+
+            var message = new MimeMessage();
+            var sender = new MailboxAddress(client.SenderName, client.SenderEmail);
+            message.From.Add(sender);
+            message.To.Add(new MailboxAddress("", email));
+            message.Subject = await Template.Parse(template.Subject).RenderAsync(template.Data);
+
+            if (!string.IsNullOrEmpty(Constants.SMTP_REPLYTO_NAME) && !string.IsNullOrEmpty(Constants.SMTP_REPLYTO_EMAIL))
+                message.ReplyTo.Add(new MailboxAddress(Constants.SMTP_REPLYTO_NAME, Constants.SMTP_REPLYTO_EMAIL));
+
+            message.Body = await GetEmailBodyAsync(template, client, sender);
+
+            await mailClient.SendAsync(message);
+        }
+
+        private async Task<MimeEntity> GetEmailBodyAsync(IEmailTemplate template, IClient client, MailboxAddress sender)
+        {
+            var builder = new BodyBuilder();
+            try
+            {
+                builder.TextBody = await Template.Parse(template.Text).RenderAsync(template.Data);
+                builder.HtmlBody = await Template.Parse(template.Html).RenderAsync(template.Data);
+
+                var key = NNGnuPGContext.GetSigningKey(sender);
+                if (key != null)
+                {
+                    using (MemoryStream outputStream = new MemoryStream())
+                    {
+                        using (Stream armoredStream = new ArmoredOutputStream(outputStream))
+                        {
+                            key.PublicKey.Encode(armoredStream);
+                        }
+                        outputStream.Seek(0, SeekOrigin.Begin);
+                        builder.Attachments.Add($"{client.Id}_pub.asc", Encoding.ASCII.GetBytes(Encoding.ASCII.GetString(outputStream.ToArray())));
+                    }
+                    return await MultipartSigned.CreateAsync(NNGnuPGContext, sender, DigestAlgorithm.Sha256, builder.ToMessageBody());
+                }
+                else
+                {
+                    return builder.ToMessageBody();
+                }
+            }
+            catch (PrivateKeyNotFoundException)
+            {
+                return builder.ToMessageBody();
+            }
+        }
+
+        async ValueTask IAsyncDisposable.DisposeAsync()
+        {
+            await mailClient.DisconnectAsync(true);
+            mailClient.Dispose();
+        }
+
+
+        static string ReadMinifiedHtmlFile(string path)
+        {
+            var settings = new HtmlMinificationSettings()
+            {
+                WhitespaceMinificationMode = WhitespaceMinificationMode.Medium
+            };
+            var cssMinifier = new KristensenCssMinifier();
+            var jsMinifier = new CrockfordJsMinifier();
+
+            var minifier = new HtmlMinifier(settings, cssMinifier, jsMinifier, new NullLogger());
+
+            return minifier.Minify(File.ReadAllText(path), false).MinifiedContent;
+        }
+    }
+
+    class NNGnuPGContext : GnuPGContext
+    {
+        IConfiguration PgpKeySettings { get; set; }
+        public NNGnuPGContext(IConfiguration pgpKeySettings)
+        {
+            PgpKeySettings = pgpKeySettings;
+        }
+
+        protected override string GetPasswordForKey(PgpSecretKey key)
+        {
+            return PgpKeySettings[key.KeyId.ToString("X")];
+        }
+    }
+}

Streetwriters.Identity/Services/MFAService.cs

@@ -39,9 +39,9 @@ namespace Streetwriters.Identity.Services
         const string SMS_ID_CLAIM = "mfa:sms:id";
 
         private UserManager<User> UserManager { get; set; }
-        private ITemplatedEmailSender EmailSender { get; set; }
+        private IEmailSender EmailSender { get; set; }
         private ISMSSender SMSSender { get; set; }
-        public MFAService(UserManager<User> _userManager, ITemplatedEmailSender emailSender, ISMSSender smsSender)
+        public MFAService(UserManager<User> _userManager, IEmailSender emailSender, ISMSSender smsSender)
         {
             UserManager = _userManager;
             EmailSender = emailSender;
@@ -54,7 +54,6 @@ namespace Streetwriters.Identity.Services
             if (!result.Succeeded) return;
 
             await this.RemovePrimaryMethodAsync(user);
-            await this.RemoveSecondaryMethodAsync(user);
             await UserManager.AddClaimAsync(user, new Claim(MFAService.PRIMARY_METHOD_CLAIM, primaryMethod));
         }
 
@@ -70,20 +69,6 @@ namespace Streetwriters.Identity.Services
             return true;
         }
 
-        public async Task<bool> ResetMFAAsync(User user)
-        {
-            await UserManager.SetTwoFactorEnabledAsync(user, false);
-            await UserManager.SetTwoFactorEnabledAsync(user, true);
-
-            await this.RemovePrimaryMethodAsync(user);
-            await this.RemoveSecondaryMethodAsync(user);
-
-            await UserManager.AddClaimAsync(user, new Claim(MFAService.PRIMARY_METHOD_CLAIM, MFAMethods.Email));
-
-            await UserManager.ResetAuthenticatorKeyAsync(user);
-            return true;
-        }
-
         public async Task SetSecondaryMethodAsync(User user, string secondaryMethod)
         {
             await this.ReplaceClaimAsync(user, MFAService.SECONDARY_METHOD_CLAIM, secondaryMethod);
@@ -97,7 +82,7 @@ namespace Streetwriters.Identity.Services
 
         public string GetPrimaryMethod(User user)
         {
-            return this.GetClaimValue(user, MFAService.PRIMARY_METHOD_CLAIM, MFAMethods.Email);
+            return this.GetClaimValue(user, MFAService.PRIMARY_METHOD_CLAIM);
         }
 
         public string GetSecondaryMethod(User user)
@@ -105,10 +90,10 @@ namespace Streetwriters.Identity.Services
             return this.GetClaimValue(user, MFAService.SECONDARY_METHOD_CLAIM);
         }
 
-        public string GetClaimValue(User user, string claimType, string defaultValue = null)
+        public string GetClaimValue(User user, string claimType)
         {
             var claim = user.Claims.FirstOrDefault((c) => c.ClaimType == claimType);
-            return claim != null ? claim.ClaimValue : defaultValue;
+            return claim != null ? claim.ClaimValue : null;
         }
 
         public Task<int> GetRemainingValidCodesAsync(User user)
@@ -121,14 +106,6 @@ namespace Streetwriters.Identity.Services
             return method == MFAMethods.App || method == MFAMethods.Email || method == MFAMethods.SMS || method == MFAMethods.RecoveryCode;
         }
 
-        public bool IsValidMFAMethod(string method, User user)
-        {
-            var primaryMethod = GetPrimaryMethod(user);
-            var secondaryMethod = GetSecondaryMethod(user);
-            if (!IsValidMFAMethod(method)) return false;
-            return method == primaryMethod || (!string.IsNullOrEmpty(secondaryMethod) && method == secondaryMethod);
-        }
-
         private Task RemoveSecondaryMethodAsync(User user)
         {
             return this.RemoveClaimAsync(user, MFAService.SECONDARY_METHOD_CLAIM);
@@ -165,8 +142,8 @@ namespace Streetwriters.Identity.Services
         public async Task SendOTPAsync(User user, IClient client, MultiFactorSetupForm form, bool isSetup = false)
         {
             var method = form.Type;
-            if ((method != MFAMethods.Email && method != MFAMethods.SMS) || !IsValidMFAMethod(method))
-                throw new Exception("Invalid method.");
+            if (method != MFAMethods.Email && method != MFAMethods.SMS) throw new Exception("Invalid method.");
+
 
             if (isSetup &&
                 method == MFAMethods.SMS &&
@@ -184,7 +161,7 @@ namespace Streetwriters.Identity.Services
                     break;
                 case "sms":
                     await UserManager.SetPhoneNumberAsync(user, form.PhoneNumber);
-                    var id = await SMSSender.SendOTPAsync(form.PhoneNumber, client);
+                    var id = SMSSender.SendOTP(form.PhoneNumber, client);
                     await this.ReplaceClaimAsync(user, MFAService.SMS_ID_CLAIM, id);
                     break;
 
@@ -197,7 +174,7 @@ namespace Streetwriters.Identity.Services
             {
                 var id = this.GetClaimValue(user, MFAService.SMS_ID_CLAIM);
                 if (string.IsNullOrEmpty(id)) throw new Exception("Could not find associated SMS verify id. Please try sending the code again.");
-                if (await SMSSender.VerifyOTPAsync(id, code))
+                if (SMSSender.VerifyOTP(id, code))
                 {
                     // Auto confirm user phone number if not confirmed
                     if (!await UserManager.IsPhoneNumberConfirmedAsync(user))

Streetwriters.Identity/Services/PasswordHasher.cs

@@ -26,19 +26,20 @@ namespace Streetwriters.Identity.Services
 {
     public class Argon2PasswordHasher<TUser> : IPasswordHasher<TUser> where TUser : User
     {
-        const long MAX_PASSWORD_LENGTH = 1024 * 2;
         public string HashPassword(TUser user, string password)
         {
-            if (password.Length > MAX_PASSWORD_LENGTH)
-                throw new Exception("Password is too long.");
-            ArgumentNullException.ThrowIfNullOrEmpty(password, nameof(password));
+            if (password == null)
+                throw new ArgumentNullException(nameof(password));
+
             return PasswordHelper.CreatePasswordHash(password);
         }
 
         public PasswordVerificationResult VerifyHashedPassword(TUser user, string hashedPassword, string providedPassword)
         {
-            ArgumentNullException.ThrowIfNullOrEmpty(hashedPassword, nameof(hashedPassword));
-            ArgumentNullException.ThrowIfNullOrEmpty(providedPassword, nameof(providedPassword));
+            if (hashedPassword == null)
+                throw new ArgumentNullException(nameof(hashedPassword));
+            if (providedPassword == null)
+                throw new ArgumentNullException(nameof(providedPassword));
 
             return PasswordHelper.VerifyPassword(providedPassword, hashedPassword) ? PasswordVerificationResult.Success : PasswordVerificationResult.Failed;
         }

Streetwriters.Identity/Services/SMSSender.cs

@@ -19,40 +19,43 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using Streetwriters.Identity.Interfaces;
 using Streetwriters.Common.Interfaces;
+using MessageBird;
+using MessageBird.Objects;
+using Microsoft.Extensions.Options;
+using Streetwriters.Identity.Models;
 using Streetwriters.Common;
-using Twilio.Rest.Verify.V2.Service;
-using Twilio;
-using System.Threading.Tasks;
 
 namespace Streetwriters.Identity.Services
 {
     public class SMSSender : ISMSSender
     {
+        private Client client;
         public SMSSender()
         {
-            if (!string.IsNullOrEmpty(Constants.TWILIO_ACCOUNT_SID) && !string.IsNullOrEmpty(Constants.TWILIO_AUTH_TOKEN))
+            if (!string.IsNullOrEmpty(Constants.MESSAGEBIRD_ACCESS_KEY))
+                client = Client.CreateDefault(Constants.MESSAGEBIRD_ACCESS_KEY);
+        }
+
+        public string SendOTP(string number, IClient app)
+        {
+            VerifyOptionalArguments optionalArguments = new VerifyOptionalArguments
             {
-                TwilioClient.Init(Constants.TWILIO_ACCOUNT_SID, Constants.TWILIO_AUTH_TOKEN);
-            }
+                Originator = app.Name,
+                Reference = app.Name,
+                Type = MessageType.Sms,
+                Template = $"Your {app.Name} 2FA code is: %token. Valid for 5 minutes.",
+                TokenLength = 6,
+                Timeout = 60 * 5
+            };
+            Verify verify = client.CreateVerify(number, optionalArguments);
+            if (verify.Status == VerifyStatus.Sent) return verify.Id;
+            return null;
         }
 
-        public async Task<string> SendOTPAsync(string number, IClient app)
+        public bool VerifyOTP(string id, string code)
         {
-            var verification = await VerificationResource.CreateAsync(
-                to: number,
-                channel: "sms",
-                pathServiceSid: Constants.TWILIO_SERVICE_SID
-            );
-            return verification.Sid;
-        }
-
-        public async Task<bool> VerifyOTPAsync(string id, string code)
-        {
-            return (await VerificationCheckResource.CreateAsync(
-                verificationSid: id,
-                pathServiceSid: Constants.TWILIO_SERVICE_SID,
-                code: code
-            )).Status == "approved";
+            Verify verify = client.SendVerifyToken(id, code);
+            return verify.Status == VerifyStatus.Verified;
         }
     }
 }

Streetwriters.Identity/Services/TemplatedEmailSender.cs

@@ -1,184 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Text;
-using System.Text.Json;
-using System.Text.Json.Serialization;
-using System.Threading;
-using System.Threading.Tasks;
-using MailKit;
-using MailKit.Net.Smtp;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Options;
-using MimeKit;
-using MimeKit.Cryptography;
-using Org.BouncyCastle.Bcpg;
-using Org.BouncyCastle.Bcpg.OpenPgp;
-using Scriban;
-using SendGrid;
-using SendGrid.Helpers.Mail;
-using Streetwriters.Common;
-using Streetwriters.Common.Helpers;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
-using Streetwriters.Identity.Interfaces;
-using Streetwriters.Identity.Models;
-using WebMarkupMin.Core;
-using WebMarkupMin.Core.Loggers;
-
-namespace Streetwriters.Identity.Services
-{
-    public class TemplatedEmailSender : ITemplatedEmailSender
-    {
-        NNGnuPGContext NNGnuPGContext { get; set; }
-        IEmailSender EmailSender { get; set; }
-
-        public TemplatedEmailSender(IConfiguration configuration, IEmailSender emailSender)
-        {
-            NNGnuPGContext = new NNGnuPGContext(configuration.GetSection("PgpKeySettings"));
-            EmailSender = emailSender;
-        }
-
-        EmailTemplate Email2FATemplate = new EmailTemplate
-        {
-            Html = HtmlHelper.ReadMinifiedHtmlFile("Templates/Email2FACode.html"),
-            Text = File.ReadAllText("Templates/Email2FACode.txt"),
-            Subject = "Your {{app_name}} account 2FA code",
-        };
-
-        EmailTemplate ConfirmEmailTemplate = new EmailTemplate
-        {
-            Html = HtmlHelper.ReadMinifiedHtmlFile("Templates/ConfirmEmail.html"),
-            Text = File.ReadAllText("Templates/ConfirmEmail.txt"),
-            Subject = "Confirm your {{app_name}} account",
-        };
-
-        EmailTemplate ConfirmChangeEmailTemplate = new EmailTemplate
-        {
-            Html = HtmlHelper.ReadMinifiedHtmlFile("Templates/EmailChangeConfirmation.html"),
-            Text = File.ReadAllText("Templates/EmailChangeConfirmation.txt"),
-            Subject = "Change {{app_name}} account email address",
-        };
-
-        EmailTemplate PasswordResetEmailTemplate = new EmailTemplate
-        {
-            Html = HtmlHelper.ReadMinifiedHtmlFile("Templates/ResetAccountPassword.html"),
-            Text = File.ReadAllText("Templates/ResetAccountPassword.txt"),
-            Subject = "Reset {{app_name}} account password",
-        };
-
-        EmailTemplate FailedLoginAlertTemplate = new EmailTemplate
-        {
-            Html = HtmlHelper.ReadMinifiedHtmlFile("Templates/FailedLoginAlert.html"),
-            Text = File.ReadAllText("Templates/FailedLoginAlert.txt"),
-            Subject = "Failed login attempt on your {{app_name}} account",
-        };
-
-        public async Task Send2FACodeEmailAsync(string email, string code, IClient client)
-        {
-            var template = new EmailTemplate
-            {
-                Html = Email2FATemplate.Html,
-                Text = Email2FATemplate.Text,
-                Subject = Email2FATemplate.Subject,
-                Data = new { app_name = client.Name, code = code },
-            };
-            await EmailSender.SendEmailAsync(email, template, client, NNGnuPGContext);
-        }
-
-        public async Task SendConfirmationEmailAsync(
-            string email,
-            string callbackUrl,
-            IClient client
-        )
-        {
-            var template = new EmailTemplate
-            {
-                Html = ConfirmEmailTemplate.Html,
-                Text = ConfirmEmailTemplate.Text,
-                Subject = ConfirmEmailTemplate.Subject,
-                Data = new { app_name = client.Name, confirm_link = callbackUrl },
-            };
-            await EmailSender.SendEmailAsync(email, template, client, NNGnuPGContext);
-        }
-
-        public async Task SendChangeEmailConfirmationAsync(
-            string email,
-            string code,
-            IClient client
-        )
-        {
-            var template = new EmailTemplate
-            {
-                Html = ConfirmChangeEmailTemplate.Html,
-                Text = ConfirmChangeEmailTemplate.Text,
-                Subject = ConfirmChangeEmailTemplate.Subject,
-                Data = new { app_name = client.Name, code = code },
-            };
-            await EmailSender.SendEmailAsync(email, template, client, NNGnuPGContext);
-        }
-
-        public async Task SendPasswordResetEmailAsync(
-            string email,
-            string callbackUrl,
-            IClient client
-        )
-        {
-            var template = new EmailTemplate
-            {
-                Html = PasswordResetEmailTemplate.Html,
-                Text = PasswordResetEmailTemplate.Text,
-                Subject = PasswordResetEmailTemplate.Subject,
-                Data = new { app_name = client.Name, reset_link = callbackUrl },
-            };
-            await EmailSender.SendEmailAsync(email, template, client, NNGnuPGContext);
-        }
-
-        public async Task SendFailedLoginAlertAsync(string email, string deviceInfo, IClient client)
-        {
-            var template = new EmailTemplate
-            {
-                Html = FailedLoginAlertTemplate.Html,
-                Text = FailedLoginAlertTemplate.Text,
-                Subject = FailedLoginAlertTemplate.Subject,
-                Data = new
-                {
-                    app_name = client.Name,
-                    device_info = deviceInfo.Replace("\n", "<br>"),
-                },
-            };
-            await EmailSender.SendEmailAsync(email, template, client, NNGnuPGContext);
-        }
-    }
-
-    public class NNGnuPGContext(IConfiguration pgpKeySettings) : GnuPGContext
-    {
-        IConfiguration PgpKeySettings { get; set; } = pgpKeySettings;
-
-        protected override string GetPasswordForKey(PgpSecretKey key)
-        {
-            return PgpKeySettings[key.KeyId.ToString("X")];
-        }
-    }
-}

Streetwriters.Identity/Services/TokenGenerationService.cs

@@ -84,13 +84,11 @@ namespace Streetwriters.Identity.Helpers
         public async Task<ClaimsPrincipal> TransformTokenRequestAsync(ValidatedTokenRequest request, User user, string grantType, string[] scopes, int lifetime = 20 * 60)
         {
             var principal = await PrincipalFactory.CreateAsync(user);
-            var identityUser = new IdentityServerUser(user.Id.ToString())
-            {
-                DisplayName = user.UserName,
-                AuthenticationTime = System.DateTime.UtcNow,
-                IdentityProvider = IdentityServerConstants.LocalIdentityProvider,
-                AdditionalClaims = principal.Claims.ToArray()
-            };
+            var identityUser = new IdentityServerUser(user.Id.ToString());
+            identityUser.DisplayName = user.UserName;
+            identityUser.AuthenticationTime = System.DateTime.UtcNow;
+            identityUser.IdentityProvider = IdentityServerConstants.LocalIdentityProvider;
+            identityUser.AdditionalClaims = principal.Claims.ToArray();
 
             request.AccessTokenType = AccessTokenType.Jwt;
             request.AccessTokenLifetime = lifetime;

Streetwriters.Identity/Services/UserAccountService.cs

@@ -1,56 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Identity;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
-using Streetwriters.Identity.Interfaces;
-using Streetwriters.Identity.Models;
-
-namespace Streetwriters.Identity.Services
-{
-    public class UserAccountService(UserManager<User> userManager, IMFAService mfaService) : IUserAccountService
-    {
-        public async Task<UserModel> GetUserAsync(string clientId, string userId)
-        {
-            var user = await userManager.FindByIdAsync(userId);
-            if (!await UserService.IsUserValidAsync(userManager, user, clientId))
-                throw new Exception($"Unable to find user with ID '{userId}'.");
-
-            var claims = await userManager.GetClaimsAsync(user);
-            var marketingConsentClaim = claims.FirstOrDefault((claim) => claim.Type == $"{clientId}:marketing_consent");
-
-            if (await userManager.IsEmailConfirmedAsync(user) && !await userManager.GetTwoFactorEnabledAsync(user))
-            {
-                await mfaService.EnableMFAAsync(user, MFAMethods.Email);
-                user = await userManager.FindByIdAsync(userId);
-            }
-
-            return new UserModel
-            {
-                UserId = user.Id.ToString(),
-                Email = user.Email,
-                IsEmailConfirmed = user.EmailConfirmed,
-                MarketingConsent = marketingConsentClaim == null,
-                MFA = new MFAConfig
-                {
-                    IsEnabled = user.TwoFactorEnabled,
-                    PrimaryMethod = mfaService.GetPrimaryMethod(user),
-                    SecondaryMethod = mfaService.GetSecondaryMethod(user),
-                    RemainingValidCodes = await mfaService.GetRemainingValidCodesAsync(user)
-                }
-            };
-        }
-
-        public async Task DeleteUserAsync(string clientId, string userId, string password)
-        {
-            var user = await userManager.FindByIdAsync(userId);
-            if (!await UserService.IsUserValidAsync(userManager, user, clientId)) throw new Exception($"User not found.");
-
-            if (!await userManager.CheckPasswordAsync(user, password)) throw new Exception("Wrong password.");
-
-            await userManager.DeleteAsync(user);
-        }
-    }
-}

Streetwriters.Identity/Services/UserService.cs

@@ -19,8 +19,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System.Linq;
 using System.Security.Claims;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Identity;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Models;
 
@@ -80,10 +78,5 @@ namespace Streetwriters.Identity.Services
         {
             return $"{clientId}:status";
         }
-
-        public static async Task<bool> IsUserValidAsync(UserManager<User> userManager, User user, string clientId)
-        {
-            return user != null && await userManager.IsInRoleAsync(user, clientId);
-        }
     }
 }