sync: replace mongodb with file system based repository

.env

@@ -6,19 +6,13 @@ SMTP_USERNAME=
 SMTP_PASSWORD=
 SMTP_HOST=
 SMTP_PORT=
-NOTESNOOK_SENDER_EMAIL= # optional
+NOTESNOOK_SENDER_EMAIL=
-NOTESNOOK_SENDER_NAME= # optional
+NOTESNOOK_SENDER_NAME=
 SMTP_REPLYTO_NAME= # optional
 SMTP_REPLYTO_EMAIL= # optional
 
-# MessageBird or Twilio are used for 2FA via SMS
+# MessageBird is used for 2FA via SMS
-# You can setup either of them or none of them but keep in mind
-# that 2FA via SMS will not work if you haven't set up at least
-# one SMS provider.
 MESSAGEBIRD_ACCESS_KEY=
-TWILIO_ACCOUNT_SID=
-TWILIO_AUTH_TOKEN=
-TWILIO_SERVICE_SID=
 
 # Server discovery settings
 # The domain must be without protocol
@@ -27,25 +21,11 @@ NOTESNOOK_SERVER_DOMAIN=
 IDENTITY_SERVER_DOMAIN=
 SSE_SERVER_DOMAIN=
 
-# Add the origins on which you want to enable CORS.
-# Leave it empty to allow all origins to access your server.
-# Seperate each origin with a comma
-# e.g. https://app.notesnook.com,http://localhost:3000
-NOTESNOOK_CORS_ORIGINS= # optional
-
 # url of the web app instance you want to use
-# e.g. https://app.notesnook.com
+# e.g. http://localhost:3000
 # Note: no slashes at the end
 NOTESNOOK_APP_HOST=
 
 # Minio is used for S3 storage
 MINIO_ROOT_USER= # aka. AccessKeyId (must be > 3 characters)
 MINIO_ROOT_PASSWORD= # aka. AccessKey (must be > 8 characters)
-
-# If you don't want to use Minio, you can use any other S3 compatible
-# storage service.
-S3_ACCESS_KEY=
-S3_ACCESS_KEY_ID=
-S3_SERVICE_URL=
-S3_REGION=
-S3_BUCKET_NAME=attachments # required

.github/workflows/publish.yml

@@ -1,85 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-# GitHub recommends pinning actions to a commit SHA.
-# To get a newer version, you will need to update the SHA.
-# You can also reference a tag or branch, but the action may change without warning.
-
-name: Publish Docker images
-
-on:
-  release:
-    types: [published]
-
-jobs:
-  push_to_registry:
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        repos:
-          - image: streetwriters/notesnook-sync
-            file: ./Notesnook.API/Dockerfile
-
-          - image: streetwriters/identity
-            file: ./Streetwriters.Identity/Dockerfile
-
-          - image: streetwriters/sse
-            file: ./Streetwriters.Messenger/Dockerfile
-    permissions:
-      packages: write
-      contents: read
-      attestations: write
-      id-token: write
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-
-      # Setup Buildx
-      - name: Docker Setup Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          ecr: auto
-          logout: true
-
-      # Pull previous image from docker hub to use it as cache to improve the image build time.
-      - name: docker pull cache image
-        continue-on-error: true
-        run: docker pull ${{ matrix.repos.image }}:latest
-
-      # Setup QEMU
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ matrix.repos.image }}
-
-      - name: Build and push Docker image
-        id: push
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ${{ matrix.repos.file }}
-          push: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v8
-          tags: ${{ steps.meta.outputs.tags }}
-          cache-from: ${{ matrix.repos.image }}:latest
-
-      - name: Generate artifact attestation
-        uses: actions/attest-build-provenance@v1
-        with:
-          subject-name: index.docker.io/${{ matrix.repos.image }}
-          subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true

.gitignore

@@ -262,6 +262,6 @@ __pycache__/
 
 keys/
 dist/
+appsettings.json
 keystore/
-.env.local
+.env.local
-Notesnook.API/sync/

.vscode/launch.json

@@ -9,7 +9,8 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-notesnook",
-      "program": "bin/Debug/net8.0/Notesnook.API.dll",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/Notesnook.API/bin/Debug/net7.0/linux-x64/Notesnook.API.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Notesnook.API",
       "stopAtEntry": false,
@@ -24,7 +25,8 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-identity",
-      "program": "bin/Debug/net8.0/Streetwriters.Identity.dll",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/Streetwriters.Identity/bin/Debug/net7.0/linux-x64/Streetwriters.Identity.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Identity",
       "stopAtEntry": false,
@@ -39,7 +41,8 @@
       "type": "coreclr",
       "request": "launch",
       "preLaunchTask": "build-messenger",
-      "program": "bin/Debug/net8.0/Streetwriters.Messenger.dll",
+      // If you have changed target frameworks, make sure to update the program path.
+      "program": "${workspaceFolder}/Streetwriters.Messenger/bin/Debug/net7.0/linux-x64/Streetwriters.Messenger.dll",
       "args": [],
       "cwd": "${workspaceFolder}/Streetwriters.Messenger",
       "stopAtEntry": false,

Notesnook.API/Accessors/SyncItemsRepositoryAccessor.cs

@@ -17,76 +17,47 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using Microsoft.Extensions.DependencyInjection;
-using MongoDB.Driver;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Repositories;
-using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Accessors
 {
     public class SyncItemsRepositoryAccessor : ISyncItemsRepositoryAccessor
     {
-        public SyncItemsRepository Notes { get; }
+        public SyncItemsRepository<Note> Notes { get; }
-        public SyncItemsRepository Notebooks { get; }
+        public SyncItemsRepository<Notebook> Notebooks { get; }
-        public SyncItemsRepository Shortcuts { get; }
+        public SyncItemsRepository<Shortcut> Shortcuts { get; }
-        public SyncItemsRepository Relations { get; }
+        public SyncItemsRepository<Relation> Relations { get; }
-        public SyncItemsRepository Reminders { get; }
+        public SyncItemsRepository<Reminder> Reminders { get; }
-        public SyncItemsRepository Contents { get; }
+        public SyncItemsRepository<Content> Contents { get; }
-        public SyncItemsRepository LegacySettings { get; }
+        public SyncItemsRepository<Setting> Settings { get; }
-        public SyncItemsRepository Settings { get; }
+        public SyncItemsRepository<Attachment> Attachments { get; }
-        public SyncItemsRepository Attachments { get; }
-        public SyncItemsRepository Colors { get; }
-        public SyncItemsRepository Vaults { get; }
-        public SyncItemsRepository Tags { get; }
         public Repository<UserSettings> UsersSettings { get; }
         public Repository<Monograph> Monographs { get; }
 
-        public SyncItemsRepositoryAccessor(IDbContext dbContext,
+        public SyncItemsRepositoryAccessor(SyncItemsRepository<Note> _notes,
-
+        SyncItemsRepository<Notebook> _notebooks,
-        [FromKeyedServices(Collections.NotebooksKey)]
+        SyncItemsRepository<Content> _content,
-        IMongoCollection<SyncItem> notebooks,
+        SyncItemsRepository<Setting> _settings,
-        [FromKeyedServices(Collections.NotesKey)]
+        SyncItemsRepository<Attachment> _attachments,
-        IMongoCollection<SyncItem> notes,
+        SyncItemsRepository<Shortcut> _shortcuts,
-        [FromKeyedServices(Collections.ContentKey)]
+        SyncItemsRepository<Relation> _relations,
-        IMongoCollection<SyncItem> content,
+        SyncItemsRepository<Reminder> _reminders,
-        [FromKeyedServices(Collections.SettingsKey)]
+        Repository<UserSettings> _usersSettings,
-        IMongoCollection<SyncItem> settings,
+        Repository<Monograph> _monographs)
-        [FromKeyedServices(Collections.LegacySettingsKey)]
-        IMongoCollection<SyncItem> legacySettings,
-        [FromKeyedServices(Collections.AttachmentsKey)]
-        IMongoCollection<SyncItem> attachments,
-        [FromKeyedServices(Collections.ShortcutsKey)]
-        IMongoCollection<SyncItem> shortcuts,
-        [FromKeyedServices(Collections.RemindersKey)]
-        IMongoCollection<SyncItem> reminders,
-        [FromKeyedServices(Collections.RelationsKey)]
-        IMongoCollection<SyncItem> relations,
-        [FromKeyedServices(Collections.ColorsKey)]
-        IMongoCollection<SyncItem> colors,
-        [FromKeyedServices(Collections.VaultsKey)]
-        IMongoCollection<SyncItem> vaults,
-        [FromKeyedServices(Collections.TagsKey)]
-        IMongoCollection<SyncItem> tags,
-
-        Repository<UserSettings> usersSettings, Repository<Monograph> monographs)
         {
-            UsersSettings = usersSettings;
+            Notebooks = _notebooks;
-            Monographs = monographs;
+            Notes = _notes;
-            Notebooks = new SyncItemsRepository(dbContext, notebooks);
+            Contents = _content;
-            Notes = new SyncItemsRepository(dbContext, notes);
+            Settings = _settings;
-            Contents = new SyncItemsRepository(dbContext, content);
+            Attachments = _attachments;
-            Settings = new SyncItemsRepository(dbContext, settings);
+            UsersSettings = _usersSettings;
-            LegacySettings = new SyncItemsRepository(dbContext, legacySettings);
+            Monographs = _monographs;
-            Attachments = new SyncItemsRepository(dbContext, attachments);
+            Shortcuts = _shortcuts;
-            Shortcuts = new SyncItemsRepository(dbContext, shortcuts);
+            Reminders = _reminders;
-            Reminders = new SyncItemsRepository(dbContext, reminders);
+            Relations = _relations;
-            Relations = new SyncItemsRepository(dbContext, relations);
-            Colors = new SyncItemsRepository(dbContext, colors);
-            Vaults = new SyncItemsRepository(dbContext, vaults);
-            Tags = new SyncItemsRepository(dbContext, tags);
         }
     }
 }

Notesnook.API/Authorization/EmailVerifiedRequirement.cs

@@ -0,0 +1,36 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authorization;
+
+namespace Notesnook.API.Authorization
+{
+    public class EmailVerifiedRequirement : AuthorizationHandler<EmailVerifiedRequirement>, IAuthorizationRequirement
+    {
+        protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, EmailVerifiedRequirement requirement)
+        {
+            var isEmailVerified = context.User.HasClaim("verified", "true");
+            var isUserBasic = context.User.HasClaim("notesnook:status", "basic") || context.User.HasClaim("notesnook:status", "premium_expired");
+            if (!isUserBasic || isEmailVerified)
+                context.Succeed(requirement);
+            return Task.CompletedTask;
+        }
+    }
+}

Notesnook.API/Authorization/ProUserRequirement.cs

@@ -17,47 +17,21 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Collections.Generic;
-using System.Diagnostics;
 using System.Linq;
-using System.Security.Claims;
-using System.Text.Json;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
 
 namespace Notesnook.API.Authorization
 {
     public class ProUserRequirement : AuthorizationHandler<ProUserRequirement>, IAuthorizationRequirement
     {
-        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
+        private string[] allowedClaims = { "trial", "premium", "premium_canceled" };
-        {
-            ["/s3"] = "upload attachments",
-            ["/s3/multipart"] = "upload attachments",
-        };
-        private readonly string[] allowedClaims = ["trial", "premium", "premium_canceled"];
         protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, ProUserRequirement requirement)
         {
-            PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
+            var isProOrTrial = context.User.HasClaim((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
-            var isProOrTrial = context.User.Claims.Any((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
+            if (isProOrTrial)
-            if (isProOrTrial) context.Succeed(requirement);
+                context.Succeed(requirement);
-            else
-            {
-                var phrase = "continue";
-                foreach (var item in pathErrorPhraseMap)
-                {
-                    if (path != null && path.StartsWithSegments(item.Key))
-                        phrase = item.Value;
-                }
-                var error = $"Please upgrade to Pro to {phrase}.";
-                context.Fail(new AuthorizationFailureReason(this, error));
-            }
             return Task.CompletedTask;
         }
-
-        public override Task HandleAsync(AuthorizationHandlerContext context)
-        {
-            return this.HandleRequirementAsync(context, this);
-        }
     }
 }

Notesnook.API/Authorization/SyncRequirement.cs

@@ -29,23 +29,27 @@ namespace Notesnook.API.Authorization
 {
     public class SyncRequirement : AuthorizationHandler<SyncRequirement>, IAuthorizationRequirement
     {
-        private readonly Dictionary<string, string> pathErrorPhraseMap = new()
+        private Dictionary<string, string> pathErrorPhraseMap = new Dictionary<string, string>
         {
             ["/sync/attachments"] = "use attachments",
             ["/sync"] = "sync your notes",
             ["/hubs/sync"] = "sync your notes",
-            ["/hubs/sync/v2"] = "sync your notes",
             ["/monographs"] = "publish monographs"
         };
 
+        private string[] allowedClaims = { "trial", "premium", "premium_canceled" };
         protected override Task HandleRequirementAsync(AuthorizationHandlerContext context, SyncRequirement requirement)
         {
             PathString path = context.Resource is DefaultHttpContext httpContext ? httpContext.Request.Path : null;
             var result = this.IsAuthorized(context.User, path);
             if (result.Succeeded) context.Succeed(requirement);
-            else if (result.AuthorizationFailure.FailureReasons.Any())
+            else
-                context.Fail(result.AuthorizationFailure.FailureReasons.First());
+            {
-            else context.Fail();
+                var hasReason = result.AuthorizationFailure.FailureReasons.Count() > 0;
+                if (hasReason)
+                    context.Fail(result.AuthorizationFailure.FailureReasons.First());
+                else context.Fail();
+            }
 
             return Task.CompletedTask;
         }
@@ -56,7 +60,7 @@ namespace Notesnook.API.Authorization
 
             if (string.IsNullOrEmpty(id))
             {
-                var reason = new[]
+                var reason = new AuthorizationFailureReason[]
                 {
                     new AuthorizationFailureReason(this, "Invalid token.")
                 };
@@ -80,7 +84,7 @@ namespace Notesnook.API.Authorization
                 }
 
                 var error = $"Please confirm your email to {phrase}.";
-                var reason = new[]
+                var reason = new AuthorizationFailureReason[]
                 {
                     new AuthorizationFailureReason(this, error)
                 };
@@ -88,6 +92,7 @@ namespace Notesnook.API.Authorization
                 //  context.Fail(new AuthorizationFailureReason(this, error));
             }
 
+            var isProOrTrial = User.HasClaim((c) => c.Type == "notesnook:status" && allowedClaims.Contains(c.Value));
             if (hasSyncScope && isInAudience && hasRole && isEmailVerified)
                 return PolicyAuthorizationResult.Success(); //(requirement);
             return PolicyAuthorizationResult.Forbid();

Notesnook.API/Constants.cs

@@ -1,18 +0,0 @@
-namespace Notesnook.API
-{
-    public class Collections
-    {
-        public const string SettingsKey = "settingsv2";
-        public const string AttachmentsKey = "attachments";
-        public const string ContentKey = "content";
-        public const string NotesKey = "notes";
-        public const string NotebooksKey = "notebooks";
-        public const string RelationsKey = "relations";
-        public const string RemindersKey = "reminders";
-        public const string LegacySettingsKey = "settings";
-        public const string ShortcutsKey = "shortcuts";
-        public const string TagsKey = "tags";
-        public const string ColorsKey = "colors";
-        public const string VaultsKey = "vaults";
-    }
-}

Notesnook.API/Controllers/AnnouncementController.cs

@@ -18,12 +18,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using MongoDB.Driver;
 using Notesnook.API.Models;
 using Streetwriters.Data.Repositories;
 
@@ -44,26 +42,10 @@ namespace Notesnook.API.Controllers
         [AllowAnonymous]
         public async Task<IActionResult> GetActiveAnnouncements([FromQuery] string userId)
         {
-            var totalActive = await Announcements.Collection.CountDocumentsAsync(Builders<Announcement>.Filter.Eq("IsActive", true));
+            var announcements = await Announcements.FindAsync((a) => a.IsActive);
-            if (totalActive <= 0) return Ok(new Announcement[] { });
+            return Ok(announcements.Where((a) => a.UserIds != null && a.UserIds.Length > 0
-
+                                                    ? a.UserIds.Contains(userId)
-            var announcements = (await Announcements.FindAsync((a) => a.IsActive)).Where((a) => a.UserIds == null || a.UserIds.Length == 0 || a.UserIds.Contains(userId));
+                                                    : true));
-            foreach (var announcement in announcements)
-            {
-                if (announcement.UserIds != null && !announcement.UserIds.Contains(userId)) continue;
-
-                foreach (var item in announcement.Body)
-                {
-                    if (item.Type != "callToActions") continue;
-                    foreach (var action in item.Actions)
-                    {
-                        if (action.Type != "link") continue;
-
-                        action.Data = action.Data.Replace("{{UserId}}", userId ?? "0");
-                    }
-                }
-            }
-            return Ok(announcements);
         }
     }
 }

Notesnook.API/Controllers/MonographsController.cs

@@ -20,11 +20,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using System;
 using System.Linq;
 using System.Security.Claims;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
-using MongoDB.Driver;
 using Notesnook.API.Models;
 using Streetwriters.Data.Interfaces;
 using Streetwriters.Data.Repositories;
@@ -76,9 +74,6 @@ namespace Notesnook.API.Controllers
         {
             if (await Monographs.GetAsync(monograph.Id) == null) return NotFound();
 
-            if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
-                return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
-
             if (monograph.EncryptedContent == null)
                 monograph.CompressedContent = monograph.Content.CompressBrotli();
             else
@@ -100,11 +95,8 @@ namespace Notesnook.API.Controllers
             var userId = this.User.FindFirstValue("sub");
             if (userId == null) return Unauthorized();
 
-            var monographs = (await Monographs.Collection.FindAsync(Builders<Monograph>.Filter.Eq("UserId", userId), new FindOptions<Monograph, ObjectWithId>
+            var userMonographs = await Monographs.FindAsync((m) => m.UserId == userId);
-            {
+            return Ok(userMonographs.Select((m) => m.Id));
-                Projection = Builders<Monograph>.Projection.Include("_id"),
-            })).ToEnumerable();
-            return Ok(monographs.Select((m) => m.Id));
         }
 
 
@@ -112,26 +104,7 @@ namespace Notesnook.API.Controllers
         [AllowAnonymous]
         public async Task<IActionResult> GetMonographAsync([FromRoute] string id)
         {
-            var monograph = await Monographs.GetAsync(id);
+            var monograph = await Monographs.FindOneAsync((m) => m.Id == id);
-            if (monograph == null)
-            {
-                return NotFound(new
-                {
-                    error = "invalid_id",
-                    error_description = $"No such monograph found."
-                });
-            }
-
-            if (monograph.EncryptedContent == null)
-                monograph.Content = monograph.CompressedContent.DecompressBrotli();
-            return Ok(monograph);
-        }
-
-        [HttpGet("{id}/destruct")]
-        [AllowAnonymous]
-        public async Task<IActionResult> DestructMonographAsync([FromRoute] string id)
-        {
-            var monograph = await Monographs.GetAsync(id);
             if (monograph == null)
             {
                 return NotFound(new
@@ -144,9 +117,12 @@ namespace Notesnook.API.Controllers
             if (monograph.SelfDestruct)
                 await Monographs.DeleteByIdAsync(monograph.Id);
 
-            return Ok();
+            if (monograph.EncryptedContent == null)
+                monograph.Content = monograph.CompressedContent.DecompressBrotli();
+            return Ok(monograph);
         }
 
+
         [HttpDelete("{id}")]
         public async Task<IActionResult> DeleteAsync([FromRoute] string id)
         {

Notesnook.API/Controllers/S3Controller.cs

@@ -29,6 +29,7 @@ namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Route("s3")]
+    [Authorize("Sync")]
     [ResponseCache(NoStore = true, Location = ResponseCacheLocation.None)]
     public class S3Controller : ControllerBase
     {
@@ -39,7 +40,6 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPut]
-        [Authorize("Pro")]
         public IActionResult Upload([FromQuery] string name)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -50,7 +50,6 @@ namespace Notesnook.API.Controllers
 
 
         [HttpGet("multipart")]
-        [Authorize("Pro")]
         public async Task<IActionResult> MultipartUpload([FromQuery] string name, [FromQuery] int parts, [FromQuery] string uploadId)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -63,7 +62,6 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpDelete("multipart")]
-        [Authorize("Pro")]
         public async Task<IActionResult> AbortMultipartUpload([FromQuery] string name, [FromQuery] string uploadId)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -76,7 +74,6 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPost("multipart")]
-        [Authorize("Pro")]
         public async Task<IActionResult> CompleteMultipartUpload([FromBody] CompleteMultipartUploadRequest uploadRequest)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -89,7 +86,7 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpGet]
-        [Authorize("Sync")]
+        [Authorize]
         public IActionResult Download([FromQuery] string name)
         {
             var userId = this.User.FindFirstValue("sub");
@@ -99,17 +96,18 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpHead]
-        [Authorize("Sync")]
+        [Authorize]
         public async Task<IActionResult> Info([FromQuery] string name)
         {
             var userId = this.User.FindFirstValue("sub");
             var size = await S3Service.GetObjectSizeAsync(userId, name);
+            if (size == null) return BadRequest();
+
             HttpContext.Response.Headers.ContentLength = size;
             return Ok();
         }
 
         [HttpDelete]
-        [Authorize("Sync")]
         public async Task<IActionResult> DeleteAsync([FromQuery] string name)
         {
             try

Notesnook.API/Controllers/SyncDeviceController.cs

@@ -1,74 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Net.Http;
-using System.Security.Claims;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.Mvc;
-using Notesnook.API.Interfaces;
-using Notesnook.API.Models.Responses;
-using Notesnook.API.Services;
-using Streetwriters.Common;
-using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Models;
-
-namespace Notesnook.API.Controllers
-{
-    [ApiController]
-    [Authorize]
-    [Route("devices")]
-    public class SyncDeviceController : ControllerBase
-    {
-        [HttpPost]
-        public async Task<IActionResult> RegisterDevice([FromQuery] string deviceId)
-        {
-            try
-            {
-                var userId = this.User.FindFirstValue("sub");
-                new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).RegisterDevice();
-                return Ok();
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't register device.", ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
-        }
-
-
-        [HttpDelete]
-        public async Task<IActionResult> UnregisterDevice([FromQuery] string deviceId)
-        {
-            try
-            {
-                var userId = this.User.FindFirstValue("sub");
-                new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).UnregisterDevice();
-                return Ok();
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(UnregisterDevice), "Couldn't unregister device.", ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
-        }
-    }
-}

Notesnook.API/Controllers/UsersController.cs

@@ -18,23 +18,35 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
+using System.Net.Http;
 using System.Security.Claims;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http.Timeouts;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Notesnook.API.Interfaces;
-using Notesnook.API.Models;
 using Notesnook.API.Models.Responses;
 using Streetwriters.Common;
+using Streetwriters.Common.Extensions;
+using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Controllers
 {
     [ApiController]
     [Authorize]
     [Route("users")]
-    public class UsersController(IUserService UserService) : ControllerBase
+    public class UsersController : ControllerBase
     {
+        private readonly HttpClient httpClient;
+        private readonly IHttpContextAccessor HttpContextAccessor;
+        private IUserService UserService { get; set; }
+        public UsersController(IUserService userService, IHttpContextAccessor accessor)
+        {
+            httpClient = new HttpClient();
+            HttpContextAccessor = accessor;
+            UserService = userService;
+        }
+
         [HttpPost]
         [AllowAnonymous]
         public async Task<IActionResult> Signup()
@@ -54,35 +66,21 @@ namespace Notesnook.API.Controllers
         [HttpGet]
         public async Task<IActionResult> GetUser()
         {
-            var userId = User.FindFirstValue("sub");
+            UserResponse response = await UserService.GetUserAsync();
-            try
+            if (!response.Success) return BadRequest(response);
-            {
+            return Ok(response);
-                UserResponse response = await UserService.GetUserAsync(userId);
-                if (!response.Success) return BadRequest(response);
-                return Ok(response);
-            }
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't get user for id.", userId, ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
         }
 
         [HttpPatch]
         public async Task<IActionResult> UpdateUser([FromBody] UserResponse user)
         {
-            var userId = User.FindFirstValue("sub");
+            UserResponse response = await UserService.GetUserAsync(false);
-            try
+
-            {
+            if (user.AttachmentsKey != null)
-                if (user.AttachmentsKey != null)
+                await UserService.SetUserAttachmentsKeyAsync(response.UserId, user.AttachmentsKey);
-                    await UserService.SetUserAttachmentsKeyAsync(userId, user.AttachmentsKey);
+            else return BadRequest();
-                return Ok();
+
-            }
+            return Ok();
-            catch (Exception ex)
-            {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't update user with id.", userId, ex.ToString());
-                return BadRequest(new { error = ex.Message });
-            }
         }
 
         [HttpPost("reset")]
@@ -96,20 +94,24 @@ namespace Notesnook.API.Controllers
         }
 
         [HttpPost("delete")]
-        [RequestTimeout(5 * 60 * 1000)]
+        public async Task<IActionResult> Delete()
-        public async Task<IActionResult> Delete([FromForm] DeleteAccountForm form)
         {
-            var userId = this.User.FindFirstValue("sub");
-            var jti = User.FindFirstValue("jti");
             try
             {
-                await UserService.DeleteUserAsync(userId, jti, form.Password);
+                var userId = this.User.FindFirstValue("sub");
-                return Ok();
+
+                if (await UserService.DeleteUserAsync(userId, User.FindFirstValue("jti")))
+                {
+                    Response response = await this.httpClient.ForwardAsync<Response>(this.HttpContextAccessor, $"{Servers.IdentityServer.ToString()}/account/unregister", HttpMethod.Post);
+                    if (!response.Success) return BadRequest();
+
+                    return Ok();
+                }
+                return BadRequest();
             }
             catch (Exception ex)
             {
-                await Slogger<UsersController>.Error(nameof(GetUser), "Couldn't delete user with id.", userId, ex.ToString());
+                return BadRequest(ex.Message);
-                return BadRequest(new { error = ex.Message });
             }
         }
     }

Notesnook.API/Dockerfile

@@ -1,52 +1,28 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build
 WORKDIR /app
-EXPOSE 80
-EXPOSE 443
-
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
-ARG TARGETARCH
-ARG BUILDPLATFORM
-ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
-
-WORKDIR /src
 
+# restore all project dependencies
 COPY Streetwriters.Data/*.csproj ./Streetwriters.Data/
+RUN dotnet restore /app/Streetwriters.Data/Streetwriters.Data.csproj --use-current-runtime
+
 COPY Streetwriters.Common/*.csproj ./Streetwriters.Common/
+RUN dotnet restore /app/Streetwriters.Common/Streetwriters.Common.csproj --use-current-runtime
+
 COPY Notesnook.API/*.csproj ./Notesnook.API/
+RUN dotnet restore /app/Notesnook.API/Notesnook.API.csproj --use-current-runtime
 
-# restore dependencies
+# copy everything else
-RUN dotnet restore -v d /src/Notesnook.API/Notesnook.API.csproj --use-current-runtime
-
 COPY Streetwriters.Data/ ./Streetwriters.Data/
 COPY Streetwriters.Common/ ./Streetwriters.Common/
 COPY Notesnook.API/ ./Notesnook.API/
 
-WORKDIR /src/Notesnook.API/
+# build
+WORKDIR /app/Notesnook.API/
+ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
+RUN dotnet publish -c Release -o /app/out --use-current-runtime --self-contained false --no-restore
 
-RUN dotnet build -c Release -o /app/build -a $TARGETARCH
+# final stage/image
-
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
-FROM build AS publish
-RUN dotnet publish -c Release -o /app/publish \
-  #--runtime alpine-x64 \
-  --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
-  /p:PublishSingleFile=true \
-  /p:JsonSerializerIsReflectionEnabledByDefault=true \
-  -a $TARGETARCH
-
-FROM --platform=$BUILDPLATFORM base AS final
-ARG TARGETARCH
-ARG BUILDPLATFORM
-
-# create a new user and change directory ownership
-RUN adduser --disabled-password \
-  --home /app \
-  --gecos '' dotnetuser && chown -R dotnetuser /app
-
-# impersonate into the new user
-USER dotnetuser
 WORKDIR /app
-
+COPY --from=build /app/out .
-COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "Notesnook.API.dll"]
-ENTRYPOINT ["./Notesnook.API"]

Notesnook.API/EventSources/SyncEventCounterSource.cs

@@ -1,49 +0,0 @@
-using System;
-using System.Diagnostics.Metrics;
-using System.Diagnostics.Tracing;
-
-[EventSource(Name = "Notesnook.API.EventCounter.Sync")]
-public sealed class SyncEventCounterSource : EventSource
-{
-    public static readonly SyncEventCounterSource Log = new();
-
-    private Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
-    private Counter<int> fetchCounter;
-    private Counter<int> pushCounter;
-    private Counter<int> legacyFetchCounter;
-    private Counter<int> pushV2Counter;
-    private Counter<int> fetchV2Counter;
-    private Histogram<long> fetchV2Duration;
-    private Histogram<long> pushV2Duration;
-    private SyncEventCounterSource()
-    {
-        fetchCounter = meter.CreateCounter<int>("sync.fetches", "fetches", "Total fetches");
-        pushCounter = meter.CreateCounter<int>("sync.pushes", "pushes", "Total pushes");
-        legacyFetchCounter = meter.CreateCounter<int>("sync.legacy-fetches", "fetches", "Total legacy fetches");
-        fetchV2Counter = meter.CreateCounter<int>("sync.v2.fetches", "fetches", "Total v2 fetches");
-        pushV2Counter = meter.CreateCounter<int>("sync.v2.pushes", "pushes", "Total v2 pushes");
-        fetchV2Duration = meter.CreateHistogram<long>("sync.v2.fetch_duration");
-        pushV2Duration = meter.CreateHistogram<long>("sync.v2.push_duration");
-    }
-
-    public void Fetch() => fetchCounter.Add(1);
-    public void LegacyFetch() => legacyFetchCounter.Add(1);
-    public void FetchV2() => fetchV2Counter.Add(1);
-    public void PushV2() => pushV2Counter.Add(1);
-    public void Push() => pushCounter.Add(1);
-    public void RecordFetchDuration(long durationMs) => fetchV2Duration.Record(durationMs);
-    public void RecordPushDuration(long durationMs) => pushV2Duration.Record(durationMs);
-
-    protected override void Dispose(bool disposing)
-    {
-        legacyFetchCounter = null;
-        fetchV2Counter = null;
-        pushV2Counter = null;
-        pushCounter = null;
-        fetchCounter = null;
-        meter.Dispose();
-        meter = null;
-
-        base.Dispose(disposing);
-    }
-}

Notesnook.API/Extensions/AuthorizationResultTransformer.cs

@@ -48,7 +48,7 @@ namespace Notesnook.API.Extensions
             {
                 var error = string.Join("\n", policyAuthorizationResult.AuthorizationFailure.FailureReasons.Select((r) => r.Message));
 
-                if (!string.IsNullOrEmpty(error))
+                if (!string.IsNullOrEmpty(error) && !isWebsocket)
                 {
                     httpContext.Response.StatusCode = (int)HttpStatusCode.Unauthorized;
                     httpContext.Response.ContentType = "application/json";

Notesnook.API/Hubs/SyncHub.cs

@@ -23,114 +23,24 @@ using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Security.Claims;
 using System.Text.Json;
-using System.Text.Json.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.SignalR;
-using MongoDB.Driver;
 using Notesnook.API.Authorization;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
-using Notesnook.API.Repositories;
 using Streetwriters.Common.Models;
 using Streetwriters.Data.Interfaces;
 
 namespace Notesnook.API.Hubs
 {
-    public struct RunningPush
-    {
-        public long Timestamp { get; set; }
-        public long Validity { get; set; }
-        public string ConnectionId { get; set; }
-    }
     public interface ISyncHubClient
     {
-        Task PushItems(SyncTransferItemV2 transferItem);
+        Task SyncItem(SyncTransferItem transferItem);
-        Task<bool> SendItems(SyncTransferItemV2 transferItem);
+        Task RemoteSyncCompleted(long lastSynced);
-        Task PushCompleted(long lastSynced);
+        Task SyncCompleted();
-    }
-
-    public class GlobalSync
-    {
-        private const long PUSH_VALIDITY_EXTENSION_PERIOD = 16 * 1000; // 16 second
-        private const int PUSH_VALIDITY_PERIOD_PER_ITEM = 5 * 100; // 0.5 second
-        private const long BASE_PUSH_VALIDITY_PERIOD = 5 * 1000; // 5 seconds
-        private const long BASE_PUSH_VALIDITY_PERIOD_NEW = 16 * 1000; // 16 seconds
-        private readonly static Dictionary<string, List<RunningPush>> PushOperations = new();
-
-        public static void ClearPushOperations(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations.ToArray())
-                    if (push.ConnectionId == connectionId || !IsPushValid(push, now))
-                        operations.Remove(push);
-            }
-        }
-
-        public static bool IsPushing(string userId, string connectionId)
-        {
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (push.ConnectionId == connectionId && IsPushValid(push, now)) return true;
-            }
-            return false;
-        }
-
-
-        public static bool IsUserPushing(string userId)
-        {
-            var count = 0;
-            if (PushOperations.TryGetValue(userId, out List<RunningPush> operations))
-            {
-                var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
-                foreach (var push in operations)
-                    if (IsPushValid(push, now)) ++count;
-            }
-            return count > 0;
-        }
-
-        public static void StartPush(string userId, string connectionId, long? totalItems = null)
-        {
-            if (IsPushing(userId, connectionId)) return;
-
-            if (!PushOperations.ContainsKey(userId))
-                PushOperations[userId] = new List<RunningPush>();
-
-            PushOperations[userId].Add(new RunningPush
-            {
-                ConnectionId = connectionId,
-                Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-                Validity = totalItems.HasValue ? BASE_PUSH_VALIDITY_PERIOD + (totalItems.Value * PUSH_VALIDITY_PERIOD_PER_ITEM) : BASE_PUSH_VALIDITY_PERIOD_NEW
-            });
-        }
-        public static void ExtendPush(string userId, string connectionId)
-        {
-            if (!IsPushing(userId, connectionId) || !PushOperations.ContainsKey(userId))
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var index = PushOperations[userId].FindIndex((push) => push.ConnectionId == connectionId);
-            if (index < 0)
-            {
-                StartPush(userId, connectionId);
-                return;
-            }
-
-            var pushOperation = PushOperations[userId][index];
-            pushOperation.Validity += PUSH_VALIDITY_EXTENSION_PERIOD;
-        }
-        private static bool IsPushValid(RunningPush push, long now)
-        {
-            return now < push.Timestamp + push.Validity;
-        }
     }
 
     [Authorize("Sync")]
@@ -138,16 +48,6 @@ namespace Notesnook.API.Hubs
     {
         private ISyncItemsRepositoryAccessor Repositories { get; }
         private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = new[] {
-            "settings",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "relation", // relations must sync at the end to prevent invalid state
-        };
 
         public SyncHub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
         {
@@ -170,235 +70,181 @@ namespace Notesnook.API.Hubs
 
         public override async Task OnDisconnectedAsync(Exception exception)
         {
-            try
+            var id = Context.User.FindFirstValue("sub");
-            {
+            await Groups.RemoveFromGroupAsync(Context.ConnectionId, id);
-                await base.OnDisconnectedAsync(exception);
+            await base.OnDisconnectedAsync(exception);
-            }
-            finally
-            {
-                var id = Context.User.FindFirstValue("sub");
-                GlobalSync.ClearPushOperations(id, Context.ConnectionId);
-            }
         }
 
-        private Action<SyncItem, string, long> MapTypeToUpsertAction(string type)
+        public async Task<int> SyncItem(BatchedSyncTransferItem transferItem)
         {
-            return type switch
-            {
-                "attachment" => Repositories.Attachments.Upsert,
-                "note" => Repositories.Notes.Upsert,
-                "notebook" => Repositories.Notebooks.Upsert,
-                "content" => Repositories.Contents.Upsert,
-                "shortcut" => Repositories.Shortcuts.Upsert,
-                "reminder" => Repositories.Reminders.Upsert,
-                "relation" => Repositories.Relations.Upsert,
-                _ => null,
-            };
-        }
-
-        public async Task<long> InitializePush(SyncMetadata syncMetadata)
-        {
-            if (syncMetadata.LastSynced <= 0) throw new HubException("Last synced time cannot be zero or less than zero.");
 
             var userId = Context.User.FindFirstValue("sub");
             if (string.IsNullOrEmpty(userId)) return 0;
 
-            UserSettings userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+            var others = Clients.OthersInGroup(userId);
-            long dateSynced = Math.Max(syncMetadata.LastSynced, userSettings.LastSynced);
 
-            GlobalSync.StartPush(userId, Context.ConnectionId);
+            UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
 
+            long dateSynced = transferItem.LastSynced > userSettings.LastSynced ? transferItem.LastSynced : userSettings.LastSynced;
 
-            if (
+            Parallel.For(0, transferItem.Items.Length, async (i) =>
-                (userSettings.VaultKey != null &&
-                syncMetadata.VaultKey != null &&
-                !userSettings.VaultKey.Equals(syncMetadata.VaultKey) &&
-                !syncMetadata.VaultKey.IsEmpty()) ||
-                (userSettings.VaultKey == null &&
-                syncMetadata.VaultKey != null &&
-                !syncMetadata.VaultKey.IsEmpty()))
             {
-                userSettings.VaultKey = syncMetadata.VaultKey;
+                var data = transferItem.Items[i];
-                await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+                var type = transferItem.Types[i];
-            }
+                var id = transferItem.Ids[i];
 
-            return dateSynced;
+                // We intentionally don't await here to speed up the sync. Fire and forget
-        }
+                // suits here because we don't really care if the item reaches the other
+                // devices.
+                others.SyncItem(
+                        new SyncTransferItem
+                        {
+                            Item = data,
+                            ItemType = type,
+                            LastSynced = dateSynced,
+                            Total = transferItem.Total,
+                            Current = transferItem.Current + i
+                        });
 
-        public async Task<int> PushItems(SyncTransferItemV2 pushItem, long dateSynced)
+                switch (type)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) return 0;
-
-            SyncEventCounterSource.Log.Push();
-
-            try
-            {
-                var others = Clients.OthersInGroup(userId);
-                others.PushItems(pushItem);
-
-                GlobalSync.ExtendPush(userId, Context.ConnectionId);
-
-                if (pushItem.Type == "settings")
                 {
-                    var settings = pushItem.Items.First();
+                    case "content":
-                    if (settings == null) return 0;
+                        await Repositories.Contents.UpsertAsync(id, data, userId, dateSynced);
-                    settings.Id = MongoDB.Bson.ObjectId.Parse(userId);
+                        break;
-                    settings.ItemId = userId;
+                    case "attachment":
-                    Repositories.LegacySettings.Upsert(settings, userId, dateSynced);
+                        await Repositories.Attachments.UpsertAsync(id, data, userId, dateSynced);
-                }
+                        break;
-                else
+                    case "note":
-                {
+                        await Repositories.Notes.UpsertAsync(id, data, userId, dateSynced);
-                    var UpsertItem = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception("Invalid item type.");
+                        break;
-                    foreach (var item in pushItem.Items)
+                    case "notebook":
-                    {
+                        await Repositories.Notebooks.UpsertAsync(id, data, userId, dateSynced);
-                        UpsertItem(item, userId, dateSynced);
+                        break;
-                    }
+                    case "shortcut":
+                        await Repositories.Shortcuts.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "reminder":
+                        await Repositories.Reminders.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "relation":
+                        await Repositories.Relations.UpsertAsync(id, data, userId, dateSynced);
+                        break;
+                    case "settings":
+                        await Repositories.Settings.UpsertAsync(userId, data, userId, dateSynced);
+                        break;
+                    case "vaultKey":
+                        userSettings.VaultKey = JsonSerializer.Deserialize<EncryptedData>(data);
+                        await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+                        break;
+                    default:
+                        throw new HubException("Invalid item type.");
                 }
+            });
 
-                return await unit.Commit() ? 1 : 0;
+            return 1;
-            }
-            catch (Exception ex)
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-                throw ex;
-            }
         }
 
         public async Task<bool> SyncCompleted(long dateSynced)
         {
             var userId = Context.User.FindFirstValue("sub");
-            try
-            {
-                UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
 
-                long lastSynced = Math.Max(dateSynced, userSettings.LastSynced);
+            UserSettings userSettings = await this.Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
 
-                userSettings.LastSynced = lastSynced;
+            long lastSynced = dateSynced > userSettings.LastSynced ? dateSynced : userSettings.LastSynced;
 
-                await this.Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
+            userSettings.LastSynced = lastSynced;
 
-                await Clients.OthersInGroup(userId).PushCompleted(lastSynced);
+            await this.Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
 
-                return true;
+            await Clients.OthersInGroup(userId).RemoteSyncCompleted(lastSynced);
-            }
+            return true;
-            finally
-            {
-                GlobalSync.ClearPushOperations(userId, Context.ConnectionId);
-            }
         }
 
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, long, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, long lastSyncedTimestamp, int size, long maxBytes, int skipChunks)
+        public async IAsyncEnumerable<SyncTransferItem> FetchItems(long lastSyncedTimestamp, [EnumeratorCancellation]
+        CancellationToken cancellationToken)
         {
-            var chunksProcessed = 0;
+            var userId = Context.User.FindFirstValue("sub");
-            for (int i = 0; i < collections.Length; i++)
+
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
+            if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
+                throw new HubException($"Provided timestamp value is too large. Server timestamp: {userSettings.LastSynced} Sent timestamp: {lastSyncedTimestamp}");
+
+            // var client = Clients.Caller;
+
+            if (lastSyncedTimestamp > 0 && userSettings.LastSynced == lastSyncedTimestamp)
             {
-                var type = types[i];
+                yield return new SyncTransferItem
-
-                using var cursor = await collections[i](userId, lastSyncedTimestamp, size);
-
-                var chunk = new List<SyncItem>();
-                long totalBytes = 0;
-                long METADATA_BYTES = 5 * 1024;
-
-                while (await cursor.MoveNextAsync())
                 {
-                    if (chunksProcessed++ < skipChunks) continue;
+                    LastSynced = userSettings.LastSynced,
-                    foreach (var item in cursor.Current)
+                    Synced = true
-                    {
+                };
-                        chunk.Add(item);
+                yield break;
-                        totalBytes += item.Length + METADATA_BYTES;
+            }
-                        if (totalBytes >= maxBytes)
-                        {
-                            yield return new SyncTransferItemV2
-                            {
-                                Items = chunk,
-                                Type = type,
-                                Count = chunksProcessed
-                            };
 
-                            totalBytes = 0;
+
-                            chunk.Clear();
+            var attachments = await Repositories.Attachments.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
-                        }
+
-                    }
+            var notes = await Repositories.Notes.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
-                }
+
-                if (chunk.Count > 0)
+            var notebooks = await Repositories.Notebooks.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var contents = await Repositories.Contents.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var settings = await Repositories.Settings.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var shortcuts = await Repositories.Shortcuts.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var reminders = await Repositories.Reminders.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var relations = await Repositories.Relations.GetItemsSyncedAfterAsync(userId, lastSyncedTimestamp);
+
+            var collections = new Dictionary<string, IEnumerable<object>>
+            {
+                ["attachment"] = attachments,
+                ["note"] = notes,
+                ["notebook"] = notebooks,
+                ["content"] = contents,
+                ["shortcut"] = shortcuts,
+                ["reminder"] = reminders,
+                ["relation"] = relations,
+                ["settings"] = settings,
+            };
+
+            if (userSettings.VaultKey != null)
+            {
+                collections.Add("vaultKey", new object[] { userSettings.VaultKey });
+            }
+
+            var total = collections.Values.Sum((a) => a.Count());
+            if (total == 0)
+            {
+                yield return new SyncTransferItem
                 {
-                    if (chunksProcessed++ < skipChunks) continue;
+                    Synced = true,
-                    yield return new SyncTransferItemV2
+                    LastSynced = userSettings.LastSynced
+                };
+                yield break;
+            }
+
+            foreach (var collection in collections)
+            {
+                foreach (var item in collection.Value)
+                {
+                    if (item == null) continue;
+                    // Check the cancellation token regularly so that the server will stop producing items if the client disconnects.
+                    cancellationToken.ThrowIfCancellationRequested();
+                    yield return new SyncTransferItem
                     {
-                        Items = chunk,
+                        LastSynced = userSettings.LastSynced,
-                        Type = type,
+                        Synced = false,
-                        Count = chunksProcessed
+                        Item = JsonSerializer.Serialize(item),
+                        ItemType = collection.Key,
+                        Total = total,
                     };
                 }
             }
         }
 
-        public Task<SyncMetadata> RequestFetch(long lastSyncedTimestamp)
-        {
-            return RequestResumableFetch(lastSyncedTimestamp);
-        }
-
-        public async Task<SyncMetadata> RequestResumableFetch(long lastSyncedTimestamp, int cursor = 0)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-
-            if (GlobalSync.IsUserPushing(userId))
-            {
-                throw new HubException("Cannot fetch data while another sync is in progress. Please try again later.");
-            }
-
-            SyncEventCounterSource.Log.Fetch();
-
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
-            if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
-            {
-                throw new HubException($"Provided timestamp value is too large. Server timestamp: {userSettings.LastSynced} Sent timestamp: {lastSyncedTimestamp}. Please run a Force Sync to fix this issue.");
-            }
-            // var client = Clients.Caller;
-
-            if (lastSyncedTimestamp > 0 && userSettings.LastSynced == lastSyncedTimestamp)
-            {
-                return new SyncMetadata
-                {
-                    LastSynced = userSettings.LastSynced,
-                };
-            }
-
-            var isResumable = lastSyncedTimestamp == 0;
-            if (!isResumable) cursor = 0;
-
-            var chunks = PrepareChunks(
-                collections: new[] {
-                    Repositories.LegacySettings.FindItemsSyncedAfter,
-                    Repositories.Attachments.FindItemsSyncedAfter,
-                    Repositories.Notes.FindItemsSyncedAfter,
-                    Repositories.Notebooks.FindItemsSyncedAfter,
-                    Repositories.Contents.FindItemsSyncedAfter,
-                    Repositories.Shortcuts.FindItemsSyncedAfter,
-                    Repositories.Reminders.FindItemsSyncedAfter,
-                    Repositories.Relations.FindItemsSyncedAfter,
-                },
-                types: CollectionKeys,
-                userId,
-                lastSyncedTimestamp,
-                size: 1000,
-                maxBytes: 7 * 1024 * 1024,
-                skipChunks: cursor
-            );
-
-            await foreach (var chunk in chunks)
-            {
-                _ = await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10));
-            }
-
-            return new SyncMetadata
-            {
-                VaultKey = userSettings.VaultKey,
-                LastSynced = userSettings.LastSynced,
-            };
-        }
     }
 
     [MessagePack.MessagePackObject]
@@ -412,6 +258,8 @@ namespace Notesnook.API.Hubs
 
         [MessagePack.Key("types")]
         public string[] Types { get; set; }
+        [MessagePack.Key("ids")]
+        public string[] Ids { get; set; }
 
         [MessagePack.Key("total")]
         public int Total { get; set; }
@@ -441,33 +289,4 @@ namespace Notesnook.API.Hubs
         [MessagePack.Key("current")]
         public int Current { get; set; }
     }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncTransferItemV2
-    {
-        [MessagePack.Key("items")]
-        [JsonPropertyName("items")]
-        public IEnumerable<SyncItem> Items { get; set; }
-
-        [MessagePack.Key("type")]
-        [JsonPropertyName("type")]
-        public string Type { get; set; }
-        [MessagePack.Key("count")]
-        [JsonPropertyName("count")]
-        public int Count { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncMetadata
-    {
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
-
-        [MessagePack.Key("lastSynced")]
-        [JsonPropertyName("lastSynced")]
-        public long LastSynced { get; set; }
-        // [MessagePack.Key("total")]
-        // public long TotalItems { get; set; }
-    }
 }

Notesnook.API/Hubs/SyncV2Hub.cs

@@ -1,311 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Diagnostics.Metrics;
-using System.Linq;
-using System.Security.Claims;
-using System.Text.Json.Serialization;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Authorization;
-using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.SignalR;
-using MongoDB.Bson;
-using MongoDB.Bson.Serialization;
-using MongoDB.Driver;
-using Notesnook.API.Authorization;
-using Notesnook.API.Interfaces;
-using Notesnook.API.Models;
-using Notesnook.API.Services;
-using Streetwriters.Data.Interfaces;
-
-namespace Notesnook.API.Hubs
-{
-    public interface ISyncV2HubClient
-    {
-        Task<bool> SendItems(SyncTransferItemV2 transferItem);
-        Task<bool> SendVaultKey(EncryptedData vaultKey);
-        Task PushCompleted();
-    }
-
-    [Authorize("Sync")]
-    public class SyncV2Hub : Hub<ISyncV2HubClient>
-    {
-        private ISyncItemsRepositoryAccessor Repositories { get; }
-        private readonly IUnitOfWork unit;
-        private readonly string[] CollectionKeys = [
-            "settingitem",
-            "attachment",
-            "note",
-            "notebook",
-            "content",
-            "shortcut",
-            "reminder",
-            "color",
-            "tag",
-            "vault",
-            "relation", // relations must sync at the end to prevent invalid state
-        ];
-
-        public SyncV2Hub(ISyncItemsRepositoryAccessor syncItemsRepositoryAccessor, IUnitOfWork unitOfWork)
-        {
-            Repositories = syncItemsRepositoryAccessor;
-            unit = unitOfWork;
-        }
-
-        public override async Task OnConnectedAsync()
-        {
-            var result = new SyncRequirement().IsAuthorized(Context.User, new PathString("/hubs/sync/v2"));
-            if (!result.Succeeded)
-            {
-                var reason = result.AuthorizationFailure.FailureReasons.FirstOrDefault();
-                throw new HubException(reason?.Message ?? "Unauthorized");
-            }
-            var id = Context.User.FindFirstValue("sub");
-            await Groups.AddToGroupAsync(Context.ConnectionId, id);
-            await base.OnConnectedAsync();
-        }
-
-        private Action<IEnumerable<SyncItem>, string, long> MapTypeToUpsertAction(string type)
-        {
-            return type switch
-            {
-                "settingitem" => Repositories.Settings.UpsertMany,
-                "attachment" => Repositories.Attachments.UpsertMany,
-                "note" => Repositories.Notes.UpsertMany,
-                "notebook" => Repositories.Notebooks.UpsertMany,
-                "content" => Repositories.Contents.UpsertMany,
-                "shortcut" => Repositories.Shortcuts.UpsertMany,
-                "reminder" => Repositories.Reminders.UpsertMany,
-                "relation" => Repositories.Relations.UpsertMany,
-                "color" => Repositories.Colors.UpsertMany,
-                "vault" => Repositories.Vaults.UpsertMany,
-                "tag" => Repositories.Tags.UpsertMany,
-                _ => null,
-            };
-        }
-
-        private Func<string, IEnumerable<string>, bool, int, Task<IAsyncCursor<SyncItem>>> MapTypeToFindItemsAction(string type)
-        {
-            return type switch
-            {
-                "settingitem" => Repositories.Settings.FindItemsById,
-                "attachment" => Repositories.Attachments.FindItemsById,
-                "note" => Repositories.Notes.FindItemsById,
-                "notebook" => Repositories.Notebooks.FindItemsById,
-                "content" => Repositories.Contents.FindItemsById,
-                "shortcut" => Repositories.Shortcuts.FindItemsById,
-                "reminder" => Repositories.Reminders.FindItemsById,
-                "relation" => Repositories.Relations.FindItemsById,
-                "color" => Repositories.Colors.FindItemsById,
-                "vault" => Repositories.Vaults.FindItemsById,
-                "tag" => Repositories.Tags.FindItemsById,
-                _ => null,
-            };
-        }
-
-        public async Task<int> PushItems(string deviceId, SyncTransferItemV2 pushItem)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
-
-            SyncEventCounterSource.Log.PushV2();
-
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-            try
-            {
-
-                var UpsertItems = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception($"Invalid item type: {pushItem.Type}.");
-                UpsertItems(pushItem.Items, userId, 1);
-
-                if (!await unit.Commit()) return 0;
-
-                await new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).AddIdsToOtherDevicesAsync(pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
-                return 1;
-            }
-            finally
-            {
-                stopwatch.Stop();
-                SyncEventCounterSource.Log.RecordPushDuration(stopwatch.ElapsedMilliseconds);
-            }
-        }
-
-        public async Task<bool> PushCompleted()
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            await Clients.OthersInGroup(userId).PushCompleted();
-            return true;
-        }
-
-        private static async IAsyncEnumerable<SyncTransferItemV2> PrepareChunks(Func<string, string[], bool, int, Task<IAsyncCursor<SyncItem>>>[] collections, string[] types, string userId, string[] ids, int size, bool resetSync, long maxBytes)
-        {
-            var chunksProcessed = 0;
-            for (int i = 0; i < collections.Length; i++)
-            {
-                var type = types[i];
-
-                var filteredIds = ids.Where((id) => id.EndsWith($":{type}")).Select((id) => id.Split(":")[0]).ToArray();
-                if (!resetSync && filteredIds.Length == 0) continue;
-
-                using var cursor = await collections[i](userId, filteredIds, resetSync, size);
-
-                var chunk = new List<SyncItem>();
-                long totalBytes = 0;
-                long METADATA_BYTES = 5 * 1024;
-
-                while (await cursor.MoveNextAsync())
-                {
-                    foreach (var item in cursor.Current)
-                    {
-                        chunk.Add(item);
-                        totalBytes += item.Length + METADATA_BYTES;
-                        if (totalBytes >= maxBytes)
-                        {
-                            yield return new SyncTransferItemV2
-                            {
-                                Items = chunk,
-                                Type = type,
-                                Count = chunksProcessed
-                            };
-
-                            totalBytes = 0;
-                            chunk.Clear();
-                        }
-                    }
-                }
-                if (chunk.Count > 0)
-                {
-                    yield return new SyncTransferItemV2
-                    {
-                        Items = chunk,
-                        Type = type,
-                        Count = chunksProcessed
-                    };
-                }
-            }
-        }
-
-        public async Task<SyncV2Metadata> RequestFetch(string deviceId)
-        {
-            var userId = Context.User.FindFirstValue("sub");
-            if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
-
-            SyncEventCounterSource.Log.FetchV2();
-
-            var deviceService = new SyncDeviceService(new SyncDevice(ref userId, ref deviceId));
-            if (!deviceService.IsDeviceRegistered()) deviceService.RegisterDevice();
-
-            var isResetSync = deviceService.IsSyncReset();
-            if (!deviceService.IsUnsynced() &&
-                !deviceService.IsSyncPending() &&
-                !isResetSync)
-                return new SyncV2Metadata { Synced = true };
-
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-            try
-            {
-                string[] ids = await deviceService.FetchUnsyncedIdsAsync();
-
-                var chunks = PrepareChunks(
-                    collections: [
-                        Repositories.Settings.FindItemsById,
-                    Repositories.Attachments.FindItemsById,
-                    Repositories.Notes.FindItemsById,
-                    Repositories.Notebooks.FindItemsById,
-                    Repositories.Contents.FindItemsById,
-                    Repositories.Shortcuts.FindItemsById,
-                    Repositories.Reminders.FindItemsById,
-                    Repositories.Colors.FindItemsById,
-                    Repositories.Tags.FindItemsById,
-                    Repositories.Vaults.FindItemsById,
-                    Repositories.Relations.FindItemsById,
-                    ],
-                    types: CollectionKeys,
-                    userId,
-                    ids,
-                    size: 1000,
-                    resetSync: isResetSync,
-                    maxBytes: 7 * 1024 * 1024
-                );
-
-                var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId.Equals(userId));
-                if (userSettings.VaultKey != null)
-                {
-                    if (!await Clients.Caller.SendVaultKey(userSettings.VaultKey).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected vault key.");
-                }
-
-                await foreach (var chunk in chunks)
-                {
-                    if (!await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected sent items.");
-
-                    if (!isResetSync)
-                    {
-                        var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
-                        ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
-                        await deviceService.WritePendingIdsAsync(ids);
-                    }
-                }
-
-                deviceService.Reset();
-
-                return new SyncV2Metadata
-                {
-                    Synced = true,
-                };
-            }
-            finally
-            {
-                stopwatch.Stop();
-                SyncEventCounterSource.Log.RecordFetchDuration(stopwatch.ElapsedMilliseconds);
-            }
-        }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncV2Metadata
-    {
-        [MessagePack.Key("synced")]
-        [JsonPropertyName("synced")]
-        public bool Synced { get; set; }
-    }
-
-    [MessagePack.MessagePackObject]
-    public struct SyncV2TransferItem
-    {
-        [MessagePack.Key("items")]
-        [JsonPropertyName("items")]
-        public IEnumerable<SyncItem> Items { get; set; }
-
-        [MessagePack.Key("type")]
-        [JsonPropertyName("type")]
-        public string Type { get; set; }
-
-        [MessagePack.Key("final")]
-        [JsonPropertyName("final")]
-        public bool Final { get; set; }
-
-        [MessagePack.Key("vaultKey")]
-        [JsonPropertyName("vaultKey")]
-        public EncryptedData VaultKey { get; set; }
-    }
-}

Notesnook.API/Interfaces/IS3Service.cs

@@ -30,7 +30,7 @@ namespace Notesnook.API.Interfaces
     {
         Task DeleteObjectAsync(string userId, string name);
         Task DeleteDirectoryAsync(string userId);
-        Task<long> GetObjectSizeAsync(string userId, string name);
+        Task<long?> GetObjectSizeAsync(string userId, string name);
         string GetUploadObjectUrl(string userId, string name);
         string GetDownloadObjectUrl(string userId, string name);
         Task<MultipartUploadMeta> StartMultipartUploadAsync(string userId, string name, int parts, string uploadId = null);

Notesnook.API/Interfaces/ISyncItem.cs

@@ -0,0 +1,43 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System.Text.Json.Serialization;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Bson.Serialization.Serializers;
+using Notesnook.API.Models;
+using Streetwriters.Common.Attributes;
+using Streetwriters.Common.Converters;
+using Streetwriters.Common.Interfaces;
+
+namespace Notesnook.API.Interfaces
+{
+    [BsonSerializer(typeof(ImpliedImplementationInterfaceSerializer<ISyncItem, SyncItem>))]
+    [JsonInterfaceConverter(typeof(InterfaceConverter<ISyncItem, SyncItem>))]
+    public interface ISyncItem
+    {
+        long DateSynced
+        {
+            get; set;
+        }
+
+        string UserId { get; set; }
+        string Algorithm { get; set; }
+        string IV { get; set; }
+    }
+}

Notesnook.API/Interfaces/ISyncItemsRepositoryAccessor.cs

@@ -26,18 +26,14 @@ namespace Notesnook.API.Interfaces
 {
     public interface ISyncItemsRepositoryAccessor
     {
-        SyncItemsRepository Notes { get; }
+        SyncItemsRepository<Note> Notes { get; }
-        SyncItemsRepository Notebooks { get; }
+        SyncItemsRepository<Notebook> Notebooks { get; }
-        SyncItemsRepository Shortcuts { get; }
+        SyncItemsRepository<Shortcut> Shortcuts { get; }
-        SyncItemsRepository Reminders { get; }
+        SyncItemsRepository<Reminder> Reminders { get; }
-        SyncItemsRepository Relations { get; }
+        SyncItemsRepository<Relation> Relations { get; }
-        SyncItemsRepository Contents { get; }
+        SyncItemsRepository<Content> Contents { get; }
-        SyncItemsRepository LegacySettings { get; }
+        SyncItemsRepository<Setting> Settings { get; }
-        SyncItemsRepository Attachments { get; }
+        SyncItemsRepository<Attachment> Attachments { get; }
-        SyncItemsRepository Settings { get; }
-        SyncItemsRepository Colors { get; }
-        SyncItemsRepository Vaults { get; }
-        SyncItemsRepository Tags { get; }
         Repository<UserSettings> UsersSettings { get; }
         Repository<Monograph> Monographs { get; }
     }

Notesnook.API/Interfaces/IUserService.cs

@@ -27,10 +27,9 @@ namespace Notesnook.API.Interfaces
     public interface IUserService
     {
         Task CreateUserAsync();
-        Task DeleteUserAsync(string userId);
+        Task<bool> DeleteUserAsync(string userId, string jti);
-        Task DeleteUserAsync(string userId, string jti, string password);
         Task<bool> ResetUserAsync(string userId, bool removeAttachments);
-        Task<UserResponse> GetUserAsync(string userId);
+        Task<UserResponse> GetUserAsync(bool repair = true);
         Task SetUserAttachmentsKeyAsync(string userId, IEncrypted key);
     }
 }

Notesnook.API/Models/Algorithms.cs

@@ -17,10 +17,17 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.Collections.Generic;
+
 namespace Notesnook.API.Models
 {
     public class Algorithms
     {
-        public static string Default => "xcha-argon2i13-7";
+        public const string Default = "xcha-argon2i13-7";
+        static readonly List<string> ALGORITHMS = new List<string> { Algorithms.Default };
+        public static bool IsValidAlgorithm(string algorithm)
+        {
+            return ALGORITHMS.Contains(algorithm);
+        }
     }
 }

Notesnook.API/Models/Announcement.cs

@@ -22,9 +22,11 @@ using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
+    [BsonCollection("notesnook", "announcements")]
     public class Announcement
     {
         public Announcement()

Notesnook.API/Models/DeleteAccountForm.cs

@@ -1,13 +0,0 @@
-using System.ComponentModel.DataAnnotations;
-
-namespace Notesnook.API.Models
-{
-    public class DeleteAccountForm
-    {
-        [Required]
-        public string Password
-        {
-            get; set;
-        }
-    }
-}

Notesnook.API/Models/EncryptedData.cs

@@ -25,10 +25,8 @@ using System.Text.Json.Serialization;
 
 namespace Notesnook.API.Models
 {
-    [MessagePack.MessagePackObject]
     public class EncryptedData : IEncrypted
     {
-        [MessagePack.Key("iv")]
         [JsonPropertyName("iv")]
         [BsonElement("iv")]
         [DataMember(Name = "iv")]
@@ -37,7 +35,6 @@ namespace Notesnook.API.Models
             get; set;
         }
 
-        [MessagePack.Key("cipher")]
         [JsonPropertyName("cipher")]
         [BsonElement("cipher")]
         [DataMember(Name = "cipher")]
@@ -46,30 +43,14 @@ namespace Notesnook.API.Models
             get; set;
         }
 
-        [MessagePack.Key("length")]
         [JsonPropertyName("length")]
         [BsonElement("length")]
         [DataMember(Name = "length")]
         public long Length { get; set; }
 
-        [MessagePack.Key("salt")]
         [JsonPropertyName("salt")]
         [BsonElement("salt")]
         [DataMember(Name = "salt")]
         public string Salt { get; set; }
-
-        public override bool Equals(object obj)
-        {
-            if (obj is EncryptedData encryptedData)
-            {
-                return IV == encryptedData.IV && Salt == encryptedData.Salt && Cipher == encryptedData.Cipher && Length == encryptedData.Length;
-            }
-            return base.Equals(obj);
-        }
-
-        public bool IsEmpty()
-        {
-            return this.Cipher == null && this.IV == null && this.Length == 0 && this.Salt == null;
-        }
     }
 }

Notesnook.API/Models/Monograph.cs

@@ -21,16 +21,11 @@ using System.Text.Json.Serialization;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Notesnook.API.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
-    public class ObjectWithId
+    [BsonCollection("notesnook", "monographs")]
-    {
-        [BsonId]
-        [BsonRepresentation(BsonType.ObjectId)]
-        public string Id { get; set; }
-    }
-
     public class Monograph : IMonograph
     {
         public Monograph()

Notesnook.API/Models/Responses/UserResponse.cs

@@ -15,9 +15,6 @@ namespace Notesnook.API.Models.Responses
         [JsonPropertyName("subscription")]
         public ISubscription Subscription { get; set; }
 
-        [JsonPropertyName("profile")]
-        public EncryptedData Profile { get; set; }
-
         [JsonIgnore]
         public bool Success { get; set; }
         public int StatusCode { get; set; }

Notesnook.API/Models/SyncItem.cs

@@ -17,24 +17,19 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System;
 using System.ComponentModel.DataAnnotations;
 using System.Runtime.Serialization;
 using System.Text.Json.Serialization;
 using MongoDB.Bson;
-using MongoDB.Bson.IO;
-using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Attributes;
-using MongoDB.Bson.Serialization.Serializers;
 using Notesnook.API.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
-    [MessagePack.MessagePackObject]
+    public class SyncItem : ISyncItem
-    public class SyncItem
     {
         [IgnoreDataMember]
-        [MessagePack.IgnoreMember]
         [JsonPropertyName("dateSynced")]
         public long DateSynced
         {
@@ -43,7 +38,6 @@ namespace Notesnook.API.Models
 
         [DataMember(Name = "userId")]
         [JsonPropertyName("userId")]
-        [MessagePack.Key("userId")]
         public string UserId
         {
             get; set;
@@ -51,7 +45,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("iv")]
         [DataMember(Name = "iv")]
-        [MessagePack.Key("iv")]
         [Required]
         public string IV
         {
@@ -61,7 +54,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("cipher")]
         [DataMember(Name = "cipher")]
-        [MessagePack.Key("cipher")]
         [Required]
         public string Cipher
         {
@@ -70,7 +62,6 @@ namespace Notesnook.API.Models
 
         [DataMember(Name = "id")]
         [JsonPropertyName("id")]
-        [MessagePack.Key("id")]
         public string ItemId
         {
             get; set;
@@ -80,7 +71,6 @@ namespace Notesnook.API.Models
         [BsonIgnoreIfDefault]
         [BsonRepresentation(BsonType.ObjectId)]
         [JsonIgnore]
-        [MessagePack.IgnoreMember]
         public ObjectId Id
         {
             get; set;
@@ -88,7 +78,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("length")]
         [DataMember(Name = "length")]
-        [MessagePack.Key("length")]
         [Required]
         public long Length
         {
@@ -97,7 +86,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("v")]
         [DataMember(Name = "v")]
-        [MessagePack.Key("v")]
         [Required]
         public double Version
         {
@@ -106,7 +94,6 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("alg")]
         [DataMember(Name = "alg")]
-        [MessagePack.Key("alg")]
         [Required]
         public string Algorithm
         {
@@ -114,100 +101,27 @@ namespace Notesnook.API.Models
         } = Algorithms.Default;
     }
 
-    public class SyncItemBsonSerializer : SerializerBase<SyncItem>
+    [BsonCollection("notesnook", "attachments")]
-    {
+    public class Attachment : SyncItem { }
-        public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, SyncItem value)
-        {
-            var writer = context.Writer;
-            writer.WriteStartDocument();
 
-            if (value.Id != ObjectId.Empty)
+    [BsonCollection("notesnook", "content")]
-            {
+    public class Content : SyncItem { }
-                writer.WriteName("_id");
-                writer.WriteObjectId(value.Id);
-            }
 
-            writer.WriteName("DateSynced");
+    [BsonCollection("notesnook", "notes")]
-            writer.WriteInt64(value.DateSynced);
+    public class Note : SyncItem { }
 
-            writer.WriteName("UserId");
+    [BsonCollection("notesnook", "notebooks")]
-            writer.WriteString(value.UserId);
+    public class Notebook : SyncItem { }
 
-            writer.WriteName("IV");
+    [BsonCollection("notesnook", "relations")]
-            writer.WriteString(value.IV);
+    public class Relation : SyncItem { }
 
-            writer.WriteName("Cipher");
+    [BsonCollection("notesnook", "reminders")]
-            writer.WriteString(value.Cipher);
+    public class Reminder : SyncItem { }
 
-            writer.WriteName("ItemId");
+    [BsonCollection("notesnook", "settings")]
-            writer.WriteString(value.ItemId);
+    public class Setting : SyncItem { }
 
-            writer.WriteName("Length");
+    [BsonCollection("notesnook", "shortcuts")]
-            writer.WriteInt64(value.Length);
+    public class Shortcut : SyncItem { }
-
-            writer.WriteName("Version");
-            writer.WriteDouble(value.Version);
-
-            writer.WriteName("Algorithm");
-            writer.WriteString(value.Algorithm);
-
-            writer.WriteEndDocument();
-        }
-
-        public override SyncItem Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
-        {
-            var bsonReader = context.Reader;
-            bsonReader.ReadStartDocument();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var id = bsonReader.ReadObjectId();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var dateSynced = bsonReader.ReadInt64();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var userId = bsonReader.ReadString();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var iv = bsonReader.ReadString();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var cipher = bsonReader.ReadString();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var itemId = bsonReader.ReadString();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var length = bsonReader.ReadInt64();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var version = bsonReader.ReadDouble();
-
-            bsonReader.ReadBsonType();
-            bsonReader.SkipName();
-            var algorithm = bsonReader.ReadString();
-
-            bsonReader.ReadEndDocument();
-            return new SyncItem
-            {
-                Id = id,
-                DateSynced = dateSynced,
-                UserId = userId,
-                IV = iv,
-                Cipher = cipher,
-                ItemId = itemId,
-                Length = length,
-                Version = version,
-                Algorithm = algorithm
-            };
-        }
-    }
 }

Notesnook.API/Models/UserSettings.cs

@@ -20,9 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Notesnook.API.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Notesnook.API.Models
 {
+    [BsonCollection("notesnook", "user_settings")]
     public class UserSettings : IUserSettings
     {
         public UserSettings()

Notesnook.API/Notesnook.API.csproj

@@ -1,23 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <StartupObject>Notesnook.API.Program</StartupObject>
+    <LangVersion>10.0</LangVersion>
+    <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
+    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AWSSDK.Core" Version="3.7.304.31" />
+    <PackageReference Include="AWSSDK.Core" Version="3.7.12.5" />
     <PackageReference Include="DotNetEnv" Version="2.3.0" />
-    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="6.2.0" />
+    <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="3.0.1" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="6.0.0" />
     <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="6.0.1-rc2.2" />
-    <PackageReference Include="AWSSDK.S3" Version="3.7.310.8" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.9.21" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="6.0.3" />
     <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Https" Version="2.2.0" />
-    <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-alpha.2" />
+ </ItemGroup>
-    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
-  </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Streetwriters.Common\Streetwriters.Common.csproj" />
@@ -25,4 +26,4 @@
   </ItemGroup>
 
 
-</Project>
+</Project>

Notesnook.API/Program.cs

@@ -26,8 +26,6 @@ using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Hosting;
 using Streetwriters.Common;
 using System.Linq;
-using Microsoft.Extensions.Logging;
-using System.Net;
 
 namespace Notesnook.API
 {
@@ -61,7 +59,6 @@ namespace Notesnook.API
                                 listenerOptions.UseHttps(Servers.NotesnookAPI.SSLCertificate);
                             });
                         }
-                        options.Listen(IPAddress.Parse("127.0.0.1"), 5067);
                     });
                 });
     }

Notesnook.API/Repositories/SyncItemsRepository.cs

@@ -19,187 +19,82 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
-using IdentityModel;
-using Microsoft.VisualBasic;
-using MongoDB.Bson;
 using MongoDB.Driver;
-using Notesnook.API.Hubs;
-using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
-using Streetwriters.Common;
+using Streetwriters.Data.Attributes;
-using Streetwriters.Data.DbContexts;
-using Streetwriters.Data.Interfaces;
-using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Repositories
 {
-    public class SyncItemsRepository : Repository<SyncItem>
+    public class SyncItemsRepository<T> where T : SyncItem
     {
-        private readonly string collectionName;
+        const string BASE_DATA_DIR = "data";
-        public SyncItemsRepository(IDbContext dbContext, IMongoCollection<SyncItem> collection) : base(dbContext, collection)
+        private string GetCollectionName()
         {
-            this.collectionName = collection.CollectionNamespace.CollectionName;
+            var attribute = (BsonCollectionAttribute)typeof(T).GetCustomAttributes(
-#if DEBUG
+                    typeof(BsonCollectionAttribute),
-            Collection.Indexes.CreateMany([
+                    true).FirstOrDefault();
-                new CreateIndexModel<SyncItem>(Builders<SyncItem>.IndexKeys.Ascending("UserId").Descending("DateSynced")),
+            if (string.IsNullOrEmpty(attribute.CollectionName) || string.IsNullOrEmpty(attribute.DatabaseName)) throw new Exception("Could not get a valid collection or database name.");
-                new CreateIndexModel<SyncItem>(Builders<SyncItem>.IndexKeys.Ascending("UserId").Ascending("ItemId")),
+            return attribute.CollectionName;
-                new CreateIndexModel<SyncItem>(Builders<SyncItem>.IndexKeys.Ascending("UserId"))
-            ]);
-#endif
         }
 
-        private readonly List<string> ALGORITHMS = [Algorithms.Default];
+        private string GetUserDirectoryPath(string userId)
-        private bool IsValidAlgorithm(string algorithm)
         {
-            return ALGORITHMS.Contains(algorithm);
+            return System.IO.Path.Join(BASE_DATA_DIR, userId, GetCollectionName());
         }
 
-        public Task<long> CountItemsSyncedAfterAsync(string userId, long timestamp)
+        private IEnumerable<string> EnumerateItems(string userId, string searchPattern = "*")
         {
-            var filter = Builders<SyncItem>.Filter.And(Builders<SyncItem>.Filter.Gt("DateSynced", timestamp), Builders<SyncItem>.Filter.Eq("UserId", userId));
+            try
-            return Collection.CountDocumentsAsync(filter);
-        }
-        public Task<IAsyncCursor<SyncItem>> FindItemsSyncedAfter(string userId, long timestamp, int batchSize)
-        {
-            var filter = Builders<SyncItem>.Filter.And(Builders<SyncItem>.Filter.Gt("DateSynced", timestamp), Builders<SyncItem>.Filter.Eq("UserId", userId));
-            return Collection.FindAsync(filter, new FindOptions<SyncItem>
             {
-                BatchSize = batchSize,
+                return System.IO.Directory.EnumerateFiles(GetUserDirectoryPath(userId), searchPattern, System.IO.SearchOption.TopDirectoryOnly);
-                AllowDiskUse = true,
+            }
-                AllowPartialResults = false,
+            catch
-                NoCursorTimeout = true,
+            {
-                Sort = new SortDefinitionBuilder<SyncItem>().Ascending("_id")
+                return new string[] { };
-            });
+            }
         }
 
-        public Task<IAsyncCursor<SyncItem>> FindItemsById(string userId, IEnumerable<string> ids, bool all, int batchSize)
+        private string FindItemById(string userId, string id)
         {
-            var filters = new List<FilterDefinition<SyncItem>>(new[] { Builders<SyncItem>.Filter.Eq("UserId", userId) });
+            try
-
-            if (!all) filters.Add(Builders<SyncItem>.Filter.In("ItemId", ids));
-
-            return Collection.FindAsync(Builders<SyncItem>.Filter.And(filters), new FindOptions<SyncItem>
             {
-                BatchSize = batchSize,
+                var files = Directory.GetFiles(GetUserDirectoryPath(userId), $"{id}-*", System.IO.SearchOption.TopDirectoryOnly);
-                AllowDiskUse = true,
+                return files.Length > 0 ? files[0] : null;
-                AllowPartialResults = false,
+            }
-                NoCursorTimeout = true
+            catch
+            {
+                return null;
+            }
+        }
+
+        public async Task<IEnumerable<string>> GetItemsSyncedAfterAsync(string userId, long timestamp)
+        {
+            var items = new List<string>();
+            await Parallel.ForEachAsync(EnumerateItems(userId), async (file, ct) =>
+            {
+                var parts = file.Split("-");
+                var id = parts[0];
+                var dateSynced = long.Parse(parts[1]);
+                if (dateSynced > timestamp) items.Add(await File.ReadAllTextAsync(file));
             });
+            return items;
         }
 
         public void DeleteByUserId(string userId)
         {
-            var filter = Builders<SyncItem>.Filter.Eq("UserId", userId);
+            Directory.Delete(GetUserDirectoryPath(userId), true);
-            var writes = new List<WriteModel<SyncItem>>
-            {
-                new DeleteManyModel<SyncItem>(filter)
-            };
-            dbContext.AddCommand((handle, ct) => Collection.BulkWriteAsync(handle, writes, options: null, ct));
         }
 
-        public void Upsert(SyncItem item, string userId, long dateSynced)
+        public async Task UpsertAsync(string id, string item, string userId, long dateSynced)
         {
-            if (item.Length > 15 * 1024 * 1024)
+            Directory.CreateDirectory(GetUserDirectoryPath(userId));
-            {
+            var oldPath = FindItemById(userId, id);
-                throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
+            var newPath = Path.Join(GetUserDirectoryPath(userId), $"{id}-{dateSynced}");
-            }
+            await File.WriteAllTextAsync(newPath, item);
-
+            if (oldPath != null) File.Delete(oldPath);
-            if (!IsValidAlgorithm(item.Algorithm))
-            {
-                throw new Exception($"Invalid alg identifier {item.Algorithm}");
-            }
-
-            // Handle case where the cipher is corrupted.
-            if (!IsBase64String(item.Cipher))
-            {
-                Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
-                throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
-            }
-
-            item.DateSynced = dateSynced;
-            item.UserId = userId;
-
-            var filter = Builders<SyncItem>.Filter.And(
-                Builders<SyncItem>.Filter.Eq("UserId", userId),
-                Builders<SyncItem>.Filter.Eq("ItemId", item.ItemId)
-            );
-
-            dbContext.AddCommand((handle, ct) => Collection.ReplaceOneAsync(handle, filter, item, new ReplaceOptions { IsUpsert = true }, ct));
-        }
-
-        public void UpsertMany(IEnumerable<SyncItem> items, string userId, long dateSynced)
-        {
-            var userIdFilter = Builders<SyncItem>.Filter.Eq("UserId", userId);
-            var writes = new List<WriteModel<SyncItem>>();
-            foreach (var item in items)
-            {
-                if (item.Length > 15 * 1024 * 1024)
-                {
-                    throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
-                }
-
-                if (!IsValidAlgorithm(item.Algorithm))
-                {
-                    throw new Exception($"Invalid alg identifier {item.Algorithm}");
-                }
-
-                // Handle case where the cipher is corrupted.
-                if (!IsBase64String(item.Cipher))
-                {
-                    Slogger<SyncHub>.Error("Upsert", "Corrupted", item.ItemId, item.Length.ToString(), item.Cipher);
-                    throw new Exception($"Corrupted item \"{item.ItemId}\" in collection \"{this.collectionName}\". Please report this error to support@streetwriters.co.");
-                }
-
-                var filter = Builders<SyncItem>.Filter.And(
-                    userIdFilter,
-                    Builders<SyncItem>.Filter.Eq("ItemId", item.ItemId)
-                );
-
-                item.DateSynced = dateSynced;
-                item.UserId = userId;
-
-                writes.Add(new ReplaceOneModel<SyncItem>(filter, item)
-                {
-                    IsUpsert = true
-                });
-            }
-            dbContext.AddCommand((handle, ct) => Collection.BulkWriteAsync(handle, writes, options: new BulkWriteOptions { IsOrdered = false }, ct));
-        }
-
-        private static bool IsBase64String(string value)
-        {
-            if (value == null || value.Length == 0 || value.Contains(' ') || value.Contains('\t') || value.Contains('\r') || value.Contains('\n'))
-                return false;
-            var index = value.Length - 1;
-            if (value[index] == '=')
-                index--;
-            if (value[index] == '=')
-                index--;
-            for (var i = 0; i <= index; i++)
-                if (IsInvalidBase64Char(value[i]))
-                    return false;
-            return true;
-        }
-
-        private static bool IsInvalidBase64Char(char value)
-        {
-            var code = (int)value;
-            // 1 - 9
-            if (code >= 48 && code <= 57)
-                return false;
-            // A - Z
-            if (code >= 65 && code <= 90)
-                return false;
-            // a - z
-            if (code >= 97 && code <= 122)
-                return false;
-            // - & _
-            return code != 45 && code != 95;
         }
     }
 }

Notesnook.API/Services/S3Service.cs

@@ -42,8 +42,7 @@ namespace Notesnook.API.Services
 
     public class S3Service : IS3Service
     {
-        private readonly string BUCKET_NAME = Constants.S3_BUCKET_NAME ?? "";
+        private readonly string BUCKET_NAME = "nn-attachments";
-        private readonly string INTERNAL_BUCKET_NAME = Constants.S3_INTERNAL_BUCKET_NAME ?? "";
         private AmazonS3Client S3Client { get; }
 
         // When running in a dockerized environment the sync server doesn't have access
@@ -97,7 +96,7 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (objectName == null) throw new Exception("Invalid object name."); ;
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(BUCKET_NAME, objectName);
 
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode)))
                 throw new Exception("Could not delete object.");
@@ -107,7 +106,7 @@ namespace Notesnook.API.Services
         {
             var request = new ListObjectsV2Request
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = BUCKET_NAME,
                 Prefix = userId,
             };
 
@@ -127,10 +126,10 @@ namespace Notesnook.API.Services
 
             if (keys.Count <= 0) return;
 
-            var deleteObjectsResponse = await GetS3Client(S3ClientMode.INTERNAL)
+            var deleteObjectsResponse = await S3Client
             .DeleteObjectsAsync(new DeleteObjectsRequest
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = BUCKET_NAME,
                 Objects = keys,
             });
 
@@ -138,14 +137,14 @@ namespace Notesnook.API.Services
                 throw new Exception("Could not delete directory.");
         }
 
-        public async Task<long> GetObjectSizeAsync(string userId, string name)
+        public async Task<long?> GetObjectSizeAsync(string userId, string name)
         {
             var url = this.GetPresignedURL(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
-            if (url == null) return 0;
+            if (url == null) return null;
 
             var request = new HttpRequestMessage(HttpMethod.Head, url);
             var response = await httpClient.SendAsync(request);
-            return response.Content.Headers.ContentLength ?? 0;
+            return response.Content.Headers.ContentLength;
         }
 
 
@@ -170,7 +169,7 @@ namespace Notesnook.API.Services
 
             if (string.IsNullOrEmpty(uploadId))
             {
-                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName);
+                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(BUCKET_NAME, objectName);
                 if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to initiate multipart upload.");
 
                 uploadId = response.UploadId;
@@ -194,7 +193,7 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
-            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(GetBucketName(S3ClientMode.INTERNAL), objectName, uploadId);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(BUCKET_NAME, objectName, uploadId);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to abort multipart upload.");
         }
 
@@ -204,7 +203,7 @@ namespace Notesnook.API.Services
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
             uploadRequest.Key = objectName;
-            uploadRequest.BucketName = GetBucketName(S3ClientMode.INTERNAL);
+            uploadRequest.BucketName = BUCKET_NAME;
             var response = await GetS3Client(S3ClientMode.INTERNAL).CompleteMultipartUploadAsync(uploadRequest);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to complete multipart upload.");
         }
@@ -216,7 +215,7 @@ namespace Notesnook.API.Services
 
             var request = new GetPreSignedUrlRequest
             {
-                BucketName = GetBucketName(mode),
+                BucketName = BUCKET_NAME,
                 Expires = System.DateTime.Now.AddHours(1),
                 Verb = httpVerb,
                 Key = objectName,
@@ -232,9 +231,9 @@ namespace Notesnook.API.Services
         private string GetPresignedURLForUploadPart(string objectName, string uploadId, int partNumber)
         {
 
-            return GetS3Client(S3ClientMode.INTERNAL).GetPreSignedURL(new GetPreSignedUrlRequest
+            return GetS3Client().GetPreSignedURL(new GetPreSignedUrlRequest
             {
-                BucketName = GetBucketName(S3ClientMode.INTERNAL),
+                BucketName = BUCKET_NAME,
                 Expires = System.DateTime.Now.AddHours(1),
                 Verb = HttpVerb.PUT,
                 Key = objectName,
@@ -264,11 +263,5 @@ namespace Notesnook.API.Services
             if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return S3InternalClient;
             return S3Client;
         }
-
-        string GetBucketName(S3ClientMode mode = S3ClientMode.EXTERNAL)
-        {
-            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return INTERNAL_BUCKET_NAME;
-            return BUCKET_NAME;
-        }
     }
 }

Notesnook.API/Services/SyncDeviceService.cs

@@ -1,223 +0,0 @@
-/*
-This file is part of the Notesnook Sync Server project (https://notesnook.com/)
-
-Copyright (C) 2023 Streetwriters (Private) Limited
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the Affero GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-Affero GNU General Public License for more details.
-
-You should have received a copy of the Affero GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace Notesnook.API.Services
-{
-    public struct SyncDevice(ref string userId, ref string deviceId)
-    {
-        public readonly string DeviceId = deviceId;
-        public readonly string UserId = userId;
-
-        private string userSyncDirectoryPath = null;
-        public string UserSyncDirectoryPath
-        {
-            get
-            {
-                userSyncDirectoryPath ??= Path.Join("sync", UserId);
-                return userSyncDirectoryPath;
-            }
-        }
-        private string userDeviceDirectoryPath = null;
-        public string UserDeviceDirectoryPath
-        {
-            get
-            {
-                userDeviceDirectoryPath ??= Path.Join(UserSyncDirectoryPath, DeviceId);
-                return userDeviceDirectoryPath;
-            }
-        }
-        private string pendingIdsFilePath = null;
-        public string PendingIdsFilePath
-        {
-            get
-            {
-                pendingIdsFilePath ??= Path.Join(UserDeviceDirectoryPath, "pending");
-                return pendingIdsFilePath;
-            }
-        }
-        private string unsyncedIdsFilePath = null;
-        public string UnsyncedIdsFilePath
-        {
-            get
-            {
-                unsyncedIdsFilePath ??= Path.Join(UserDeviceDirectoryPath, "unsynced");
-                return unsyncedIdsFilePath;
-            }
-        }
-        private string resetSyncFilePath = null;
-        public string ResetSyncFilePath
-        {
-            get
-            {
-                resetSyncFilePath ??= Path.Join(UserDeviceDirectoryPath, "reset-sync");
-                return resetSyncFilePath;
-            }
-        }
-    }
-    public class SyncDeviceService(SyncDevice device)
-    {
-        public async Task<string[]> GetUnsyncedIdsAsync()
-        {
-            try
-            {
-                return await File.ReadAllLinesAsync(device.UnsyncedIdsFilePath);
-            }
-            catch { return []; }
-        }
-
-        public async Task<string[]> GetUnsyncedIdsAsync(string deviceId)
-        {
-            try
-            {
-                return await File.ReadAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, deviceId, "unsynced"));
-            }
-            catch { return []; }
-        }
-
-        public async Task<string[]> FetchUnsyncedIdsAsync()
-        {
-            if (IsSyncReset()) return Array.Empty<string>();
-            if (UnsyncedIdsFileLocks.TryGetValue(device.DeviceId, out SemaphoreSlim fileLock) && fileLock.CurrentCount == 0)
-                await fileLock.WaitAsync();
-            try
-            {
-                var unsyncedIds = await GetUnsyncedIdsAsync();
-                if (IsSyncPending())
-                {
-                    unsyncedIds = unsyncedIds.Union(await File.ReadAllLinesAsync(device.PendingIdsFilePath)).ToArray();
-                }
-
-                if (unsyncedIds.Length == 0) return [];
-
-                File.Delete(device.UnsyncedIdsFilePath);
-                await File.WriteAllLinesAsync(device.PendingIdsFilePath, unsyncedIds);
-
-                return unsyncedIds;
-            }
-            catch
-            {
-                return Array.Empty<string>();
-            }
-            finally
-            {
-                if (fileLock != null && fileLock.CurrentCount == 0) fileLock.Release();
-            }
-        }
-
-
-        public async Task WritePendingIdsAsync(IEnumerable<string> ids)
-        {
-            await File.WriteAllLinesAsync(device.PendingIdsFilePath, ids);
-        }
-
-        public bool IsSyncReset()
-        {
-            return File.Exists(device.ResetSyncFilePath);
-        }
-        public bool IsSyncReset(string deviceId)
-        {
-            return File.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId, "reset-sync"));
-        }
-
-        public bool IsSyncPending()
-        {
-            return File.Exists(device.PendingIdsFilePath);
-        }
-
-        public bool IsUnsynced()
-        {
-            return File.Exists(device.UnsyncedIdsFilePath);
-        }
-
-        public void Reset()
-        {
-            File.Delete(device.ResetSyncFilePath);
-            File.Delete(device.PendingIdsFilePath);
-        }
-
-        public bool IsDeviceRegistered()
-        {
-            return Directory.Exists(device.UserDeviceDirectoryPath);
-        }
-        public bool IsDeviceRegistered(string deviceId)
-        {
-            return Directory.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId));
-        }
-
-        public string[] ListDevices()
-        {
-            return Directory.GetDirectories(device.UserSyncDirectoryPath).Select((path) => path[(path.LastIndexOf(Path.DirectorySeparatorChar) + 1)..]).ToArray();
-        }
-
-        public void ResetDevices()
-        {
-            if (File.Exists(device.UserSyncDirectoryPath)) File.Delete(device.UserSyncDirectoryPath);
-            Directory.CreateDirectory(device.UserSyncDirectoryPath);
-        }
-
-        private readonly ConcurrentDictionary<string, SemaphoreSlim> UnsyncedIdsFileLocks = [];
-        public async Task AddIdsToOtherDevicesAsync(List<string> ids)
-        {
-            await Parallel.ForEachAsync(ListDevices(), async (id, ct) =>
-            {
-                if (id == device.DeviceId || IsSyncReset(id)) return;
-                if (!UnsyncedIdsFileLocks.TryGetValue(id, out SemaphoreSlim fileLock))
-                {
-                    fileLock = UnsyncedIdsFileLocks.AddOrUpdate(id, (id) => new SemaphoreSlim(1, 1), (id, old) => new SemaphoreSlim(1, 1));
-                }
-
-                await fileLock.WaitAsync(ct);
-                try
-                {
-                    if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
-
-                    var oldIds = await GetUnsyncedIdsAsync(id);
-                    await File.WriteAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds), ct);
-                }
-                finally
-                {
-                    fileLock.Release();
-                }
-            });
-        }
-
-        public void RegisterDevice()
-        {
-            Directory.CreateDirectory(device.UserDeviceDirectoryPath);
-            File.Create(device.ResetSyncFilePath).Close();
-        }
-
-        public void UnregisterDevice()
-        {
-            try
-            {
-                Directory.Delete(device.UserDeviceDirectoryPath, true);
-            }
-            catch { }
-        }
-    }
-}

Notesnook.API/Services/UserService.cs

@@ -18,9 +18,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.IO;
 using System.Net.Http;
-using System.Text;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
@@ -65,8 +63,7 @@ namespace Notesnook.API.Services
             if (!response.Success || (response.Errors != null && response.Errors.Length > 0))
             {
                 await Slogger<UserService>.Error(nameof(CreateUserAsync), "Couldn't sign up.", JsonSerializer.Serialize(response));
-                if (response.Errors != null && response.Errors.Length > 0)
+                if (response.Errors != null && response.Errors.Length > 0) throw new Exception(string.Join(" ", response.Errors));
-                    throw new Exception(string.Join(" ", response.Errors));
                 else throw new Exception("Could not create a new account. Error code: " + response.StatusCode);
             }
 
@@ -79,7 +76,7 @@ namespace Notesnook.API.Services
 
             if (!Constants.IS_SELF_HOSTED)
             {
-                await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.CreateSubscriptionTopic, new CreateSubscriptionMessage
+                await WampServers.SubscriptionServer.PublishMessageAsync(WampServers.SubscriptionServer.Topics.CreateSubscriptionTopic, new CreateSubscriptionMessage
                 {
                     AppId = ApplicationType.NOTESNOOK,
                     Provider = SubscriptionProvider.STREETWRITERS,
@@ -92,11 +89,10 @@ namespace Notesnook.API.Services
             await Slogger<UserService>.Info(nameof(CreateUserAsync), "New user created.", JsonSerializer.Serialize(response));
         }
 
-        public async Task<UserResponse> GetUserAsync(string userId)
+        public async Task<UserResponse> GetUserAsync(bool repair = true)
         {
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
+            UserResponse response = await httpClient.ForwardAsync<UserResponse>(this.HttpContextAccessor, $"{Servers.IdentityServer.ToString()}/account", HttpMethod.Get);
-
+            if (!response.Success) return response;
-            var user = await userService.GetUserAsync(Clients.Notesnook.Id, userId) ?? throw new Exception("User not found.");
 
             ISubscription subscription = null;
             if (Constants.IS_SELF_HOSTED)
@@ -106,7 +102,7 @@ namespace Notesnook.API.Services
                     AppId = ApplicationType.NOTESNOOK,
                     Provider = SubscriptionProvider.STREETWRITERS,
                     Type = SubscriptionType.PREMIUM,
-                    UserId = user.UserId,
+                    UserId = response.UserId,
                     StartDate = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
                     // this date doesn't matter as the subscription is static.
                     ExpiryDate = DateTimeOffset.UtcNow.AddYears(1).ToUnixTimeMilliseconds()
@@ -114,38 +110,61 @@ namespace Notesnook.API.Services
             }
             else
             {
-                var subscriptionService = await WampServers.SubscriptionServer.GetServiceAsync<IUserSubscriptionService>(SubscriptionServerTopics.UserSubscriptionServiceTopic);
+                SubscriptionResponse subscriptionResponse = await httpClient.ForwardAsync<SubscriptionResponse>(this.HttpContextAccessor, $"{Servers.SubscriptionServer}/subscriptions", HttpMethod.Get);
-                subscription = await subscriptionService.GetUserSubscriptionAsync(Clients.Notesnook.Id, userId);
+                if (repair && subscriptionResponse.StatusCode == 404)
+                {
+                    await Slogger<UserService>.Error(nameof(GetUserAsync), "Repairing user subscription.", JsonSerializer.Serialize(response));
+                    // user was partially created. We should continue the process here.
+                    await WampServers.SubscriptionServer.PublishMessageAsync(WampServers.SubscriptionServer.Topics.CreateSubscriptionTopic, new CreateSubscriptionMessage
+                    {
+                        AppId = ApplicationType.NOTESNOOK,
+                        Provider = SubscriptionProvider.STREETWRITERS,
+                        Type = SubscriptionType.TRIAL,
+                        UserId = response.UserId,
+                        StartTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                        ExpiryTime = DateTimeOffset.UtcNow.AddDays(7).ToUnixTimeMilliseconds()
+                    });
+                    // just a dummy object
+                    subscriptionResponse.Subscription = new Subscription
+                    {
+                        AppId = ApplicationType.NOTESNOOK,
+                        Provider = SubscriptionProvider.STREETWRITERS,
+                        Type = SubscriptionType.TRIAL,
+                        UserId = response.UserId,
+                        StartDate = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
+                        ExpiryDate = DateTimeOffset.UtcNow.AddDays(7).ToUnixTimeMilliseconds()
+                    };
+                }
+                subscription = subscriptionResponse.Subscription;
             }
 
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == user.UserId) ?? throw new Exception("User settings not found.");
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == response.UserId);
-            return new UserResponse
+            if (repair && userSettings == null)
             {
-                UserId = user.UserId,
+                await Slogger<UserService>.Error(nameof(GetUserAsync), "Repairing user settings.", JsonSerializer.Serialize(response));
-                Email = user.Email,
+                userSettings = new UserSettings
-                IsEmailConfirmed = user.IsEmailConfirmed,
+                {
-                MarketingConsent = user.MarketingConsent,
+                    UserId = response.UserId,
-                MFA = user.MFA,
+                    LastSynced = 0,
-                PhoneNumber = user.PhoneNumber,
+                    Salt = GetSalt()
-                AttachmentsKey = userSettings.AttachmentsKey,
+                };
-                Salt = userSettings.Salt,
+                await Repositories.UsersSettings.InsertAsync(userSettings);
-                Subscription = subscription,
+            }
-                Success = true,
+            response.AttachmentsKey = userSettings.AttachmentsKey;
-                StatusCode = 200
+            response.Salt = userSettings.Salt;
-            };
+            response.Subscription = subscription;
+            return response;
         }
 
         public async Task SetUserAttachmentsKeyAsync(string userId, IEncrypted key)
         {
-            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId) ?? throw new Exception("User not found.");
+            var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
             userSettings.AttachmentsKey = (EncryptedData)key;
             await Repositories.UsersSettings.UpdateAsync(userSettings.Id, userSettings);
         }
 
-        public async Task DeleteUserAsync(string userId)
+        public async Task<bool> DeleteUserAsync(string userId, string jti)
         {
-            new SyncDeviceService(new SyncDevice(ref userId, ref userId)).ResetDevices();
-
             var cc = new CancellationTokenSource();
 
             Repositories.Notes.DeleteByUserId(userId);
@@ -153,59 +172,40 @@ namespace Notesnook.API.Services
             Repositories.Shortcuts.DeleteByUserId(userId);
             Repositories.Contents.DeleteByUserId(userId);
             Repositories.Settings.DeleteByUserId(userId);
-            Repositories.LegacySettings.DeleteByUserId(userId);
             Repositories.Attachments.DeleteByUserId(userId);
             Repositories.Reminders.DeleteByUserId(userId);
             Repositories.Relations.DeleteByUserId(userId);
-            Repositories.Colors.DeleteByUserId(userId);
-            Repositories.Tags.DeleteByUserId(userId);
-            Repositories.Vaults.DeleteByUserId(userId);
             Repositories.UsersSettings.Delete((u) => u.UserId == userId);
             Repositories.Monographs.DeleteMany((m) => m.UserId == userId);
 
-            var result = await unit.Commit();
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "User data deleted", userId, result.ToString());
-            if (!result) throw new Exception("Could not delete user data.");
-
             if (!Constants.IS_SELF_HOSTED)
             {
-                await WampServers.SubscriptionServer.PublishMessageAsync(SubscriptionServerTopics.DeleteSubscriptionTopic, new DeleteSubscriptionMessage
+                await WampServers.SubscriptionServer.PublishMessageAsync(WampServers.SubscriptionServer.Topics.DeleteSubscriptionTopic, new DeleteSubscriptionMessage
                 {
                     AppId = ApplicationType.NOTESNOOK,
                     UserId = userId
                 });
             }
 
-            await S3Service.DeleteDirectoryAsync(userId);
+            await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
-        }
-
-        public async Task DeleteUserAsync(string userId, string jti, string password)
-        {
-            await Slogger<UserService>.Info(nameof(DeleteUserAsync), "Deleting user account", userId);
-
-            var userService = await WampServers.IdentityServer.GetServiceAsync<IUserAccountService>(IdentityServerTopics.UserAccountServiceTopic);
-            await userService.DeleteUserAsync(Clients.Notesnook.Id, userId, password);
-
-            await DeleteUserAsync(userId);
-
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
             {
                 SendToAll = false,
                 OriginTokenId = jti,
                 UserId = userId,
                 Message = new Message
                 {
-                    Type = "logout",
+                    Type = "userDeleted",
-                    Data = JsonSerializer.Serialize(new { reason = "Account deleted." })
+                    Data = JsonSerializer.Serialize(new { reason = "accountDeleted" })
                 }
             });
 
+            await S3Service.DeleteDirectoryAsync(userId);
+
+            return await unit.Commit();
         }
 
         public async Task<bool> ResetUserAsync(string userId, bool removeAttachments)
         {
-            new SyncDeviceService(new SyncDevice(ref userId, ref userId)).ResetDevices();
-
             var cc = new CancellationTokenSource();
 
             Repositories.Notes.DeleteByUserId(userId);
@@ -213,13 +213,9 @@ namespace Notesnook.API.Services
             Repositories.Shortcuts.DeleteByUserId(userId);
             Repositories.Contents.DeleteByUserId(userId);
             Repositories.Settings.DeleteByUserId(userId);
-            Repositories.LegacySettings.DeleteByUserId(userId);
             Repositories.Attachments.DeleteByUserId(userId);
             Repositories.Reminders.DeleteByUserId(userId);
             Repositories.Relations.DeleteByUserId(userId);
-            Repositories.Colors.DeleteByUserId(userId);
-            Repositories.Tags.DeleteByUserId(userId);
-            Repositories.Vaults.DeleteByUserId(userId);
             Repositories.Monographs.DeleteMany((m) => m.UserId == userId);
             if (!await unit.Commit()) return false;
 
@@ -237,7 +233,7 @@ namespace Notesnook.API.Services
             return true;
         }
 
-        private static string GetSalt()
+        private string GetSalt()
         {
             byte[] salt = new byte[16];
             Rng.GetNonZeroBytes(salt);

Notesnook.API/Startup.cs

@@ -34,7 +34,6 @@ using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Connections;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.ResponseCompression;
-using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -51,8 +50,6 @@ using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Notesnook.API.Repositories;
 using Notesnook.API.Services;
-using OpenTelemetry.Metrics;
-using OpenTelemetry.Resources;
 using Streetwriters.Common;
 using Streetwriters.Common.Extensions;
 using Streetwriters.Common.Messages;
@@ -76,11 +73,12 @@ namespace Notesnook.API
         // This method gets called by the runtime. Use this method to add services to the container.
         public void ConfigureServices(IServiceCollection services)
         {
-            services.AddSingleton(MongoDbContext.CreateMongoDbClient(new DbSettings
+            var dbSettings = new DbSettings
             {
                 ConnectionString = Constants.MONGODB_CONNECTION_STRING,
                 DatabaseName = Constants.MONGODB_DATABASE_NAME
-            }));
+            };
+            services.AddSingleton<IDbSettings>(dbSettings);
 
             services.TryAddSingleton<IHttpContextAccessor, HttpContextAccessor>();
 
@@ -108,13 +106,23 @@ namespace Notesnook.API
                     policy.RequireAuthenticatedUser();
                     policy.Requirements.Add(new SyncRequirement());
                 });
+                options.AddPolicy("Verified", policy =>
+                {
+                    policy.AuthenticationSchemes.Add("introspection");
+                    policy.RequireAuthenticatedUser();
+                    policy.Requirements.Add(new EmailVerifiedRequirement());
+                });
                 options.AddPolicy("Pro", policy =>
                 {
                     policy.AuthenticationSchemes.Add("introspection");
                     policy.RequireAuthenticatedUser();
-                    policy.Requirements.Add(new SyncRequirement());
                     policy.Requirements.Add(new ProUserRequirement());
                 });
+                options.AddPolicy("BasicAdmin", policy =>
+                {
+                    policy.AuthenticationSchemes.Add("BasicAuthentication");
+                    policy.RequireClaim(ClaimTypes.Role, "Admin");
+                });
 
                 options.DefaultPolicy = options.GetPolicy("Notesnook");
             }).AddSingleton<IAuthorizationMiddlewareResultHandler, AuthorizationResultTransformer>(); ;
@@ -144,55 +152,48 @@ namespace Notesnook.API
                     context.HttpContext.User = context.Principal;
                     return Task.CompletedTask;
                 };
-                options.CacheKeyGenerator = (options, token) => (token + ":" + "reference_token").Sha256();
                 options.SaveToken = true;
                 options.EnableCaching = true;
                 options.CacheDuration = TimeSpan.FromMinutes(30);
             });
 
-            BsonSerializer.RegisterSerializer(new SyncItemBsonSerializer());
             if (!BsonClassMap.IsClassMapRegistered(typeof(UserSettings)))
+            {
                 BsonClassMap.RegisterClassMap<UserSettings>();
+            }
 
             if (!BsonClassMap.IsClassMapRegistered(typeof(EncryptedData)))
+            {
                 BsonClassMap.RegisterClassMap<EncryptedData>();
+            }
 
             if (!BsonClassMap.IsClassMapRegistered(typeof(CallToAction)))
+            {
                 BsonClassMap.RegisterClassMap<CallToAction>();
+            }
+
+            if (!BsonClassMap.IsClassMapRegistered(typeof(Announcement)))
+            {
+                BsonClassMap.RegisterClassMap<Announcement>();
+            }
 
             services.AddScoped<IDbContext, MongoDbContext>();
             services.AddScoped<IUnitOfWork, UnitOfWork>();
+            services.AddScoped(typeof(Repository<>));
+            services.AddScoped(typeof(SyncItemsRepository<>));
 
-            services.AddRepository<UserSettings>("user_settings", "notesnook")
+            services.TryAddTransient<ISyncItemsRepositoryAccessor, SyncItemsRepositoryAccessor>();
-                    .AddRepository<Monograph>("monographs", "notesnook")
+            services.TryAddTransient<IUserService, UserService>();
-                    .AddRepository<Announcement>("announcements", "notesnook");
+            services.TryAddTransient<IS3Service, S3Service>();
-
-            services.AddMongoCollection(Collections.SettingsKey)
-                    .AddMongoCollection(Collections.AttachmentsKey)
-                    .AddMongoCollection(Collections.ContentKey)
-                    .AddMongoCollection(Collections.NotesKey)
-                    .AddMongoCollection(Collections.NotebooksKey)
-                    .AddMongoCollection(Collections.RelationsKey)
-                    .AddMongoCollection(Collections.RemindersKey)
-                    .AddMongoCollection(Collections.LegacySettingsKey)
-                    .AddMongoCollection(Collections.ShortcutsKey)
-                    .AddMongoCollection(Collections.TagsKey)
-                    .AddMongoCollection(Collections.ColorsKey)
-                    .AddMongoCollection(Collections.VaultsKey);
-
-            services.AddScoped<ISyncItemsRepositoryAccessor, SyncItemsRepositoryAccessor>();
-            services.AddScoped<IUserService, UserService>();
-            services.AddScoped<IS3Service, S3Service>();
 
             services.AddControllers();
 
-            services.AddHealthChecks(); // .AddMongoDb(dbSettings.ConnectionString, dbSettings.DatabaseName, "database-check");
+            services.AddHealthChecks().AddMongoDb(dbSettings.ConnectionString, dbSettings.DatabaseName, "database-check");
             services.AddSignalR((hub) =>
             {
                 hub.MaximumReceiveMessageSize = 100 * 1024 * 1024;
-                hub.ClientTimeoutInterval = TimeSpan.FromMinutes(10);
                 hub.EnableDetailedErrors = true;
-            }).AddMessagePackProtocol().AddJsonProtocol();
+            }).AddMessagePackProtocol();
 
             services.AddResponseCompression(options =>
             {
@@ -209,13 +210,6 @@ namespace Notesnook.API
             {
                 options.Level = CompressionLevel.Fastest;
             });
-
-            services.AddOpenTelemetry()
-                    .ConfigureResource(resource => resource
-                        .AddService(serviceName: "Notesnook.API"))
-                    .WithMetrics((builder) => builder
-                            .AddMeter("Notesnook.API.Metrics.Sync")
-                            .AddPrometheusExporter());
         }
 
         // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
@@ -229,24 +223,17 @@ namespace Notesnook.API
                 });
             }
 
-            app.UseOpenTelemetryPrometheusScrapingEndpoint((context) => context.Request.Path == "/metrics" && context.Connection.LocalPort == 5067);
             app.UseResponseCompression();
 
             app.UseCors("notesnook");
-            app.UseVersion(Servers.NotesnookAPI);
+            app.UseVersion();
 
             app.UseWamp(WampServers.NotesnookServer, (realm, server) =>
             {
-                realm.Subscribe<DeleteUserMessage>(IdentityServerTopics.DeleteUserTopic, async (ev) =>
+                IUserService service = app.GetScopedService<IUserService>();
+                realm.Subscribe<DeleteUserMessage>(server.Topics.DeleteUserTopic, async (ev) =>
                 {
-                    IUserService service = app.GetScopedService<IUserService>();
+                    await service.DeleteUserAsync(ev.UserId, null);
-                    await service.DeleteUserAsync(ev.UserId);
-                });
-
-                realm.Subscribe<ClearCacheMessage>(IdentityServerTopics.ClearCacheTopic, (ev) =>
-                {
-                    IDistributedCache cache = app.GetScopedService<IDistributedCache>();
-                    ev.Keys.ForEach((key) => cache.Remove(key));
                 });
             });
 
@@ -257,7 +244,6 @@ namespace Notesnook.API
 
             app.UseEndpoints(endpoints =>
             {
-                endpoints.MapPrometheusScrapingEndpoint();
                 endpoints.MapControllers();
                 endpoints.MapHealthChecks("/health");
                 endpoints.MapHub<SyncHub>("/hubs/sync", options =>
@@ -265,21 +251,7 @@ namespace Notesnook.API
                     options.CloseOnAuthenticationExpiration = false;
                     options.Transports = HttpTransportType.WebSockets;
                 });
-                endpoints.MapHub<SyncV2Hub>("/hubs/sync/v2", options =>
-                {
-                    options.CloseOnAuthenticationExpiration = false;
-                    options.Transports = HttpTransportType.WebSockets;
-                });
             });
         }
     }
-
-    public static class ServiceCollectionMongoCollectionExtensions
-    {
-        public static IServiceCollection AddMongoCollection(this IServiceCollection services, string collectionName, string database = "notesnook")
-        {
-            services.AddKeyedSingleton(collectionName, (provider, key) => MongoDbContext.GetMongoCollection<SyncItem>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
-            return services;
-        }
-    }
 }

Notesnook.API/appsettings.Development.json

@@ -3,9 +3,7 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information",
+      "Microsoft.Hosting.Lifetime": "Information"
-      "Microsoft.AspNetCore.SignalR": "Trace",
-      "Microsoft.AspNetCore.Http.Connections": "Trace"
     }
   },
   "MongoDbSettings": {

Notesnook.API/appsettings.json

@@ -1,7 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Warning"
-    }
-  }
-}

README.md

@@ -8,7 +8,7 @@ This repo contains the full source code of the Notesnook Sync Server licensed un
 
 Requirements:
 
-1. [.NET 8](https://dotnet.microsoft.com/en-us/download/dotnet/8.0)
+1. [.NET 7](https://dotnet.microsoft.com/en-us/download/dotnet/7.0)
 2. [git](https://git-scm.com/downloads)
 
 The first step is to `clone` the repository:
@@ -55,14 +55,19 @@ dotnet run --project Streetwriters.Identity/Streetwriters.Identity.csproj
 
 The sync server can easily be started using Docker.
 
+The first step is to `clone` the repository:
+
 ```bash
-wget https://raw.githubusercontent.com/streetwriters/notesnook-sync-server/master/docker-compose.yml
+git clone https://github.com/streetwriters/notesnook-sync-server.git
+
+# change directory
+cd notesnook-sync-server
 ```
 
 And then use Docker Compose to start the servers:
 
 ```bash
-docker compose up
+docker-compose up
 ```
 
 This takes care of setting up everything including MongoDB, Minio etc.
@@ -76,7 +81,7 @@ This takes care of setting up everything including MongoDB, Minio etc.
 - [x] Open source the SSE Messaging infrastructure
 - [x] Fully Dockerize all services
 - [x] Use self-hosted Minio for S3 storage
-- [x] Publish on DockerHub
+- [ ] Publish on DockerHub
 - [ ] Write self hosting docs
 - [ ] Add settings to change server URLs in Notesnook client apps
 

Streetwriters.Common/Clients.cs

@@ -29,7 +29,7 @@ namespace Streetwriters.Common
 {
     public class Clients
     {
-        public static readonly Client Notesnook = new()
+        private static Client Notesnook = new Client
         {
             Id = "notesnook",
             Name = "Notesnook",
@@ -41,7 +41,7 @@ namespace Streetwriters.Common
             EmailConfirmedRedirectURL = $"{Constants.NOTESNOOK_APP_HOST}/account/verified",
             OnEmailConfirmed = async (userId) =>
             {
-                await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
+                await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
                 {
                     UserId = userId,
                     Message = new Message
@@ -53,7 +53,7 @@ namespace Streetwriters.Common
             }
         };
 
-        public static Dictionary<string, Client> ClientsMap = new()
+        public static Dictionary<string, Client> ClientsMap = new Dictionary<string, Client>
         {
             { "notesnook", Notesnook }
         };

Streetwriters.Common/Constants.cs

@@ -24,17 +24,12 @@ namespace Streetwriters.Common
     public class Constants
     {
         public static bool IS_SELF_HOSTED => Environment.GetEnvironmentVariable("SELF_HOSTED") == "1";
-        public static bool DISABLE_ACCOUNT_CREATION => Environment.GetEnvironmentVariable("DISABLE_ACCOUNT_CREATION") == "1";
-        public static string INSTANCE_NAME => Environment.GetEnvironmentVariable("INSTANCE_NAME") ?? "default";
 
         // S3 related
         public static string S3_ACCESS_KEY => Environment.GetEnvironmentVariable("S3_ACCESS_KEY");
         public static string S3_ACCESS_KEY_ID => Environment.GetEnvironmentVariable("S3_ACCESS_KEY_ID");
         public static string S3_SERVICE_URL => Environment.GetEnvironmentVariable("S3_SERVICE_URL");
         public static string S3_REGION => Environment.GetEnvironmentVariable("S3_REGION");
-        public static string S3_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_BUCKET_NAME");
-        public static string S3_INTERNAL_BUCKET_NAME => Environment.GetEnvironmentVariable("S3_INTERNAL_BUCKET_NAME");
-        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
 
         // SMTP settings
         public static string SMTP_USERNAME => Environment.GetEnvironmentVariable("SMTP_USERNAME");
@@ -50,23 +45,22 @@ namespace Streetwriters.Common
         public static string NOTESNOOK_API_SECRET => Environment.GetEnvironmentVariable("NOTESNOOK_API_SECRET");
 
         // MessageBird is used for SMS sending
-        public static string TWILIO_ACCOUNT_SID => Environment.GetEnvironmentVariable("TWILIO_ACCOUNT_SID");
+        public static string MESSAGEBIRD_ACCESS_KEY => Environment.GetEnvironmentVariable("MESSAGEBIRD_ACCESS_KEY");
-        public static string TWILIO_AUTH_TOKEN => Environment.GetEnvironmentVariable("TWILIO_AUTH_TOKEN");
+
-        public static string TWILIO_SERVICE_SID => Environment.GetEnvironmentVariable("TWILIO_SERVICE_SID");
         // Server discovery
-        public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT") ?? "80");
+        public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT"));
         public static string NOTESNOOK_SERVER_HOST => Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_HOST");
         public static string NOTESNOOK_SERVER_DOMAIN => Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_DOMAIN");
         public static string NOTESNOOK_CERT_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_PATH");
         public static string NOTESNOOK_CERT_KEY_PATH => Environment.GetEnvironmentVariable("NOTESNOOK_CERT_KEY_PATH");
 
-        public static int IDENTITY_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("IDENTITY_SERVER_PORT") ?? "80");
+        public static int IDENTITY_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("IDENTITY_SERVER_PORT"));
         public static string IDENTITY_SERVER_HOST => Environment.GetEnvironmentVariable("IDENTITY_SERVER_HOST");
         public static string IDENTITY_SERVER_DOMAIN => Environment.GetEnvironmentVariable("IDENTITY_SERVER_DOMAIN");
         public static string IDENTITY_CERT_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_PATH");
         public static string IDENTITY_CERT_KEY_PATH => Environment.GetEnvironmentVariable("IDENTITY_CERT_KEY_PATH");
 
-        public static int SSE_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SSE_SERVER_PORT") ?? "80");
+        public static int SSE_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SSE_SERVER_PORT"));
         public static string SSE_SERVER_HOST => Environment.GetEnvironmentVariable("SSE_SERVER_HOST");
         public static string SSE_SERVER_DOMAIN => Environment.GetEnvironmentVariable("SSE_SERVER_DOMAIN");
         public static string SSE_CERT_PATH => Environment.GetEnvironmentVariable("SSE_CERT_PATH");
@@ -75,7 +69,8 @@ namespace Streetwriters.Common
         // internal
         public static string MONGODB_CONNECTION_STRING => Environment.GetEnvironmentVariable("MONGODB_CONNECTION_STRING");
         public static string MONGODB_DATABASE_NAME => Environment.GetEnvironmentVariable("MONGODB_DATABASE_NAME");
-        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_PORT") ?? "80");
+        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
+        public static int SUBSCRIPTIONS_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_PORT"));
         public static string SUBSCRIPTIONS_SERVER_HOST => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_HOST");
         public static string SUBSCRIPTIONS_SERVER_DOMAIN => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_SERVER_DOMAIN");
         public static string SUBSCRIPTIONS_CERT_PATH => Environment.GetEnvironmentVariable("SUBSCRIPTIONS_CERT_PATH");

Streetwriters.Common/Enums/SubscriptionType.cs

@@ -26,7 +26,6 @@ namespace Streetwriters.Common.Enums
         BETA = 2,
         PREMIUM = 5,
         PREMIUM_EXPIRED = 6,
-        PREMIUM_CANCELED = 7,
+        PREMIUM_CANCELED = 7
-        PREMIUM_PAUSED = 8
     }
 }

Streetwriters.Common/Extensions/AppBuilderExtensions.cs

@@ -18,8 +18,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using System;
-using System.Collections.Generic;
-using System.Text.Json;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.DependencyInjection;
@@ -32,20 +30,13 @@ namespace Streetwriters.Common.Extensions
 {
     public static class AppBuilderExtensions
     {
-        public static IApplicationBuilder UseVersion(this IApplicationBuilder app, Server server)
+        public static IApplicationBuilder UseVersion(this IApplicationBuilder app)
         {
             app.Map("/version", (app) =>
             {
                 app.Run(async context =>
                 {
-                    context.Response.ContentType = "application/json";
+                    await context.Response.WriteAsync(Version.AsString());
-                    var data = new Dictionary<string, string>
-                    {
-                        { "version", Version.AsString() },
-                        { "id", server.Id },
-                        { "instance", Constants.INSTANCE_NAME }
-                    };
-                    await context.Response.WriteAsync(JsonSerializer.Serialize(data));
                 });
             });
             return app;

Streetwriters.Common/Extensions/ServiceCollectionServiceExtensions.cs

@@ -18,20 +18,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 using Microsoft.Extensions.DependencyInjection;
-using Streetwriters.Data.DbContexts;
-using Streetwriters.Data.Repositories;
 
 namespace Streetwriters.Common.Extensions
 {
     public static class ServiceCollectionServiceExtensions
     {
-        public static IServiceCollection AddRepository<T>(this IServiceCollection services, string collectionName, string database) where T : class
-        {
-            services.AddSingleton((provider) => MongoDbContext.GetMongoCollection<T>(provider.GetService<MongoDB.Driver.IMongoClient>(), database, collectionName));
-            services.AddScoped<Repository<T>>();
-            return services;
-        }
-
         public static IServiceCollection AddDefaultCors(this IServiceCollection services)
         {
             services.AddCors(options =>

Streetwriters.Common/Extensions/StringExtensions.cs

@@ -26,11 +26,15 @@ namespace System
 {
     public static class StringExtensions
     {
-        public static string Sha256(this string input)
+        public static string ToSha256(this string rawData, int maxLength = 12)
         {
-            var bytes = Encoding.UTF8.GetBytes(input);
+            // Create a SHA256   
-            var hash = SHA256.HashData(bytes);
+            using (SHA256 sha256Hash = SHA256.Create())
-            return Convert.ToBase64String(hash);
+            {
+                // ComputeHash - returns byte array  
+                byte[] bytes = sha256Hash.ComputeHash(Encoding.UTF8.GetBytes(rawData));
+                return ToHex(bytes, 0, maxLength);
+            }
         }
 
         public static byte[] CompressBrotli(this string input)

Streetwriters.Common/Helpers/WampHelper.cs

@@ -27,9 +27,9 @@ namespace Streetwriters.Common.Helpers
 {
     public class WampHelper
     {
-        public static async Task<IWampRealmProxy> OpenWampChannelAsync(string server, string realmName)
+        public static async Task<IWampRealmProxy> OpenWampChannelAsync<T>(string server, string realmName)
         {
-            DefaultWampChannelFactory channelFactory = new();
+            DefaultWampChannelFactory channelFactory = new DefaultWampChannelFactory();
 
             IWampChannel channel = channelFactory.CreateJsonChannel(server, realmName);
 

Streetwriters.Common/Interfaces/IUserAccountService.cs

@@ -1,16 +0,0 @@
-using System.Threading.Tasks;
-using Streetwriters.Common.Models;
-using WampSharp.V2.Rpc;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IUserAccountService
-    {
-        [WampProcedure("co.streetwriters.identity.users.get_user")]
-        Task<UserModel> GetUserAsync(string clientId, string userId);
-        [WampProcedure("co.streetwriters.identity.users.delete_user")]
-        Task DeleteUserAsync(string clientId, string userId, string password);
-        // [WampProcedure("co.streetwriters.identity.users.create_user")]
-        // Task<UserModel> CreateUserAsync();
-    }
-}

Streetwriters.Common/Interfaces/IUserSubscriptionService.cs

@@ -1,13 +0,0 @@
-using System.Threading.Tasks;
-using Streetwriters.Common.Helpers;
-using Streetwriters.Common.Models;
-using WampSharp.V2.Rpc;
-
-namespace Streetwriters.Common.Interfaces
-{
-    public interface IUserSubscriptionService
-    {
-        [WampProcedure("co.streetwriters.subscriptions.subscriptions.get_user_subscription")]
-        Task<Subscription> GetUserSubscriptionAsync(string clientId, string userId);
-    }
-}

Streetwriters.Common/Models/Offer.cs

@@ -26,9 +26,11 @@ using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("subscriptions", "offers")]
     public class Offer : IOffer
     {
         public Offer()

Streetwriters.Common/Models/Role.cs

@@ -20,9 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
 using AspNetCore.Identity.Mongo.Model;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("identity", "roles")]
     public class Role : MongoRole
     {
         //     [DataMember(Name = "email")]

Streetwriters.Common/Models/Subscription.cs

@@ -24,9 +24,11 @@ using MongoDB.Bson;
 using MongoDB.Bson.Serialization.Attributes;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Interfaces;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("subscriptions", "subscriptions")]
     public class Subscription : ISubscription
     {
         public Subscription()

Streetwriters.Common/Models/User.cs

@@ -20,9 +20,11 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
 using AspNetCore.Identity.Mongo.Model;
+using Streetwriters.Data.Attributes;
 
 namespace Streetwriters.Common.Models
 {
+    [BsonCollection("identity", "users")]
     public class User : MongoUser
     {
     }

Streetwriters.Common/Models/UserModel.cs

@@ -35,9 +35,6 @@ namespace Streetwriters.Common.Models
         [JsonPropertyName("isEmailConfirmed")]
         public bool IsEmailConfirmed { get; set; }
 
-        [JsonPropertyName("marketingConsent")]
-        public bool MarketingConsent { get; set; }
-
         [JsonPropertyName("mfa")]
         public MFAConfig MFA { get; set; }
     }

Streetwriters.Common/Servers.cs

@@ -34,7 +34,7 @@ namespace Streetwriters.Common
             if (!string.IsNullOrEmpty(originCertPath) && !string.IsNullOrEmpty(originCertKeyPath))
                 this.SSLCertificate = X509Certificate2.CreateFromPemFile(originCertPath, originCertKeyPath);
         }
-        public string Id { get; set; }
+
         public int Port { get; set; }
         public string Hostname { get; set; }
         public string Domain { get; set; }
@@ -63,13 +63,13 @@ namespace Streetwriters.Common
     public class Servers
     {
 #if DEBUG
-        public static string GetLocalIPv4()
+        public static string GetLocalIPv4(NetworkInterfaceType _type)
         {
             var interfaces = NetworkInterface.GetAllNetworkInterfaces();
             string output = "";
             foreach (NetworkInterface item in interfaces)
             {
-                if ((item.NetworkInterfaceType == NetworkInterfaceType.Ethernet || item.NetworkInterfaceType == NetworkInterfaceType.Wireless80211) && item.OperationalStatus == OperationalStatus.Up)
+                if (item.NetworkInterfaceType == _type && item.OperationalStatus == OperationalStatus.Up)
                 {
                     foreach (UnicastIPAddressInformation ip in item.GetIPProperties().UnicastAddresses)
                     {
@@ -82,7 +82,7 @@ namespace Streetwriters.Common
             }
             return output;
         }
-        public readonly static string HOST = GetLocalIPv4();
+        public readonly static string HOST = GetLocalIPv4(NetworkInterfaceType.Ethernet);
         public static Server S3Server { get; } = new()
         {
             Port = 4568,
@@ -95,7 +95,6 @@ namespace Streetwriters.Common
             Domain = Constants.NOTESNOOK_SERVER_DOMAIN,
             Port = Constants.NOTESNOOK_SERVER_PORT,
             Hostname = Constants.NOTESNOOK_SERVER_HOST,
-            Id = "notesnook-sync"
         };
 
         public static Server MessengerServer { get; } = new(Constants.SSE_CERT_PATH, Constants.SSE_CERT_KEY_PATH)
@@ -103,7 +102,6 @@ namespace Streetwriters.Common
             Domain = Constants.SSE_SERVER_DOMAIN,
             Port = Constants.SSE_SERVER_PORT,
             Hostname = Constants.SSE_SERVER_HOST,
-            Id = "sse"
         };
 
         public static Server IdentityServer { get; } = new(Constants.IDENTITY_CERT_PATH, Constants.IDENTITY_CERT_KEY_PATH)
@@ -111,7 +109,6 @@ namespace Streetwriters.Common
             Domain = Constants.IDENTITY_SERVER_DOMAIN,
             Port = Constants.IDENTITY_SERVER_PORT,
             Hostname = Constants.IDENTITY_SERVER_HOST,
-            Id = "auth"
         };
 
         public static Server SubscriptionServer { get; } = new(Constants.SUBSCRIPTIONS_CERT_PATH, Constants.SUBSCRIPTIONS_CERT_KEY_PATH)
@@ -119,7 +116,6 @@ namespace Streetwriters.Common
             Domain = Constants.SUBSCRIPTIONS_SERVER_DOMAIN,
             Port = Constants.SUBSCRIPTIONS_SERVER_PORT,
             Hostname = Constants.SUBSCRIPTIONS_SERVER_HOST,
-            Id = "subscription"
         };
     }
 }

Streetwriters.Common/Streetwriters.Common.csproj

@@ -1,10 +1,10 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
-
+  
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Cors" Version="2.2.0" />
@@ -15,8 +15,8 @@
     <PackageReference Include="AspNetCore.Identity.Mongo" Version="8.3.3" />
   </ItemGroup>
 
-
+  
   <ItemGroup>
     <ProjectReference Include="..\Streetwriters.Data\Streetwriters.Data.csproj" />
   </ItemGroup>
-</Project>
+</Project>

Streetwriters.Common/Version.cs

@@ -21,8 +21,8 @@ namespace Streetwriters.Common
 {
     public class Version
     {
-        public const int MAJOR = 1;
+        public const int MAJOR = 2;
-        public const int MINOR = 0;
+        public const int MINOR = 3;
         public const int PATCH = 0;
         public static string AsString()
         {

Streetwriters.Common/WampServers.cs

@@ -23,7 +23,6 @@ using System.Collections.Generic;
 using System.Reactive.Subjects;
 using System.Threading.Tasks;
 using Streetwriters.Common.Helpers;
-using Streetwriters.Common.Interfaces;
 using WampSharp.V2.Client;
 
 namespace Streetwriters.Common
@@ -37,28 +36,25 @@ namespace Streetwriters.Common
         public T Topics { get; set; } = new T();
         public string Realm { get; set; }
 
-        private async Task<IWampRealmProxy> GetChannelAsync(string topic)
-        {
-            if (!Channels.TryGetValue(topic, out IWampRealmProxy channel) || !channel.Monitor.IsConnected)
-            {
-                channel = await WampHelper.OpenWampChannelAsync(Address, Realm);
-                Channels.AddOrUpdate(topic, (key) => channel, (key, old) => channel);
-            }
-            return channel;
-        }
-
-        public async Task<V> GetServiceAsync<V>(string topic) where V : class
-        {
-            var channel = await GetChannelAsync(topic);
-            return channel.Services.GetCalleeProxy<V>();
-        }
-
         public async Task PublishMessageAsync<V>(string topic, V message)
         {
             try
             {
-                IWampRealmProxy channel = await GetChannelAsync(topic);
+                IWampRealmProxy channel;
-                WampHelper.PublishMessage(channel, topic, message);
+                if (Channels.ContainsKey(topic))
+                    channel = Channels[topic];
+                else
+                {
+                    channel = await WampHelper.OpenWampChannelAsync<V>(this.Address, this.Realm);
+                    Channels.TryAdd(topic, channel);
+                }
+                if (!channel.Monitor.IsConnected)
+                {
+                    Channels.TryRemove(topic, out IWampRealmProxy value);
+                    await PublishMessageAsync<V>(topic, message);
+                    return;
+                }
+                WampHelper.PublishMessage<V>(channel, topic, message);
             }
             catch (Exception ex)
             {
@@ -101,25 +97,23 @@ namespace Streetwriters.Common
 
     public class MessengerServerTopics
     {
-        public const string SendSSETopic = "co.streetwriters.sse.send";
+        public string SendSSETopic => "com.streetwriters.sse.send";
     }
 
     public class SubscriptionServerTopics
     {
-        public const string UserSubscriptionServiceTopic = "co.streetwriters.subscriptions.subscriptions";
+        public string CreateSubscriptionTopic => "com.streetwriters.subscriptions.create";
-
+        public string DeleteSubscriptionTopic => "com.streetwriters.subscriptions.delete";
-        public const string CreateSubscriptionTopic = "co.streetwriters.subscriptions.create";
-        public const string DeleteSubscriptionTopic = "co.streetwriters.subscriptions.delete";
     }
 
     public class IdentityServerTopics
     {
-        public const string UserAccountServiceTopic = "co.streetwriters.identity.users";
+        public string CreateSubscriptionTopic => "com.streetwriters.subscriptions.create";
-        public const string ClearCacheTopic = "co.streetwriters.identity.clear_cache";
+        public string DeleteSubscriptionTopic => "com.streetwriters.subscriptions.delete";
-        public const string DeleteUserTopic = "co.streetwriters.identity.delete_user";
     }
 
     public class NotesnookServerTopics
     {
+        public string DeleteUserTopic => "com.streetwriters.notesnook.user.delete";
     }
 }

Streetwriters.Data/Attributes/BsonCollection.cs

@@ -0,0 +1,36 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+using System;
+
+namespace Streetwriters.Data.Attributes
+{
+    [AttributeUsage(AttributeTargets.Class, Inherited = false)]
+    public class BsonCollectionAttribute : Attribute
+    {
+        public string CollectionName { get; }
+        public string DatabaseName { get; }
+
+        public BsonCollectionAttribute(string databaseName, string collectionName)
+        {
+            CollectionName = collectionName;
+            DatabaseName = databaseName;
+        }
+    }
+}

Streetwriters.Data/DbContexts/MongoDbContext.cs

@@ -28,26 +28,20 @@ using System.Threading.Tasks;
 
 namespace Streetwriters.Data.DbContexts
 {
-    public class MongoDbContext(IMongoClient MongoClient) : IDbContext
+    public class MongoDbContext : IDbContext
     {
-        public static IMongoClient CreateMongoDbClient(IDbSettings dbSettings)
+        private IMongoDatabase Database { get; set; }
+        private MongoClient MongoClient { get; set; }
+        private readonly List<Func<IClientSessionHandle, CancellationToken, Task>> _commands;
+        private IDbSettings DbSettings { get; set; }
+        public MongoDbContext(IDbSettings dbSettings)
         {
-            var settings = MongoClientSettings.FromConnectionString(dbSettings.ConnectionString);
+            DbSettings = dbSettings;
-            settings.MaxConnectionPoolSize = 500;
+            Configure();
-            settings.MinConnectionPoolSize = 0;
+            // Every command will be stored and it'll be processed at SaveChanges
-            return new MongoClient(settings);
+            _commands = new List<Func<IClientSessionHandle, CancellationToken, Task>>();
         }
 
-        public static IMongoCollection<T> GetMongoCollection<T>(IMongoClient client, string databaseName, string collectionName)
-        {
-            return client.GetDatabase(databaseName).GetCollection<T>(collectionName, new MongoCollectionSettings()
-            {
-                AssignIdOnInsert = true,
-            });
-        }
-
-        private readonly List<Func<IClientSessionHandle, CancellationToken, Task>> _commands = [];
-
         public async Task<int> SaveChanges()
         {
             try
@@ -57,7 +51,7 @@ namespace Streetwriters.Data.DbContexts
                 using (IClientSessionHandle session = await MongoClient.StartSessionAsync())
                 {
 #if DEBUG
-                    await Parallel.ForEachAsync(_commands, async (c, ct) => await c(session, ct));
+                    await Task.WhenAll(_commands.Select(c => c(session, default(CancellationToken))));
 #else
                     await session.WithTransactionAsync(async (handle, token) =>
                     {
@@ -77,6 +71,26 @@ namespace Streetwriters.Data.DbContexts
             }
         }
 
+        private void Configure()
+        {
+            if (MongoClient != null)
+            {
+                return;
+            }
+            var settings = MongoClientSettings.FromConnectionString(DbSettings.ConnectionString);
+            settings.MaxConnectionPoolSize = 5000;
+            settings.MinConnectionPoolSize = 300;
+            MongoClient = new MongoClient(settings);
+        }
+
+        public IMongoCollection<T> GetCollection<T>(string databaseName, string collectionName)
+        {
+            return MongoClient.GetDatabase(databaseName).GetCollection<T>(collectionName, new MongoCollectionSettings()
+            {
+                AssignIdOnInsert = true,
+            });
+        }
+
         public void AddCommand(Func<IClientSessionHandle, CancellationToken, Task> func)
         {
             _commands.Add(func);
@@ -86,5 +100,10 @@ namespace Streetwriters.Data.DbContexts
         {
             GC.SuppressFinalize(this);
         }
+
+        public Task DropDatabaseAsync()
+        {
+            return MongoClient.DropDatabaseAsync(DbSettings.DatabaseName);
+        }
     }
 }

Streetwriters.Data/Interfaces/IDbContext.cs

@@ -29,5 +29,6 @@ namespace Streetwriters.Data.Interfaces
     {
         void AddCommand(Func<IClientSessionHandle, CancellationToken, Task> func);
         Task<int> SaveChanges();
+        IMongoCollection<T> GetCollection<T>(string databaseName, string collectionName);
     }
 }

Streetwriters.Data/Repositories/Repository.cs

@@ -24,6 +24,7 @@ using System.Linq.Expressions;
 using System.Threading.Tasks;
 using MongoDB.Bson;
 using MongoDB.Driver;
+using Streetwriters.Data.Attributes;
 using Streetwriters.Data.Interfaces;
 
 namespace Streetwriters.Data.Repositories
@@ -31,14 +32,24 @@ namespace Streetwriters.Data.Repositories
     public class Repository<TEntity> where TEntity : class
     {
         protected readonly IDbContext dbContext;
-        public IMongoCollection<TEntity> Collection { get; set; }
+        protected IMongoCollection<TEntity> Collection { get; set; }
 
-        public Repository(IDbContext _dbContext, IMongoCollection<TEntity> collection)
+        public Repository(IDbContext _dbContext)
         {
             dbContext = _dbContext;
-            Collection = collection;
+            Collection = GetCollection();
         }
 
+        private protected IMongoCollection<TEntity> GetCollection()
+        {
+            var attribute = (BsonCollectionAttribute)typeof(TEntity).GetCustomAttributes(
+                    typeof(BsonCollectionAttribute),
+                    true).FirstOrDefault();
+            if (string.IsNullOrEmpty(attribute.CollectionName) || string.IsNullOrEmpty(attribute.DatabaseName)) throw new Exception("Could not get a valid collection or database name.");
+            return dbContext.GetCollection<TEntity>(attribute.DatabaseName, attribute.CollectionName);
+        }
+
+
         public virtual void Insert(TEntity obj)
         {
             dbContext.AddCommand((handle, ct) => Collection.InsertOneAsync(handle, obj, null, ct));

Streetwriters.Data/Streetwriters.Data.csproj

@@ -1,15 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
   </PropertyGroup>
 
-
+    
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.0" />
-    <PackageReference Include="MongoDB.Driver" Version="2.22.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.13.2" />
-    <PackageReference Include="MongoDB.Driver.Core" Version="2.22.0" />
+    <PackageReference Include="MongoDB.Driver.Core" Version="2.13.2" />
-    <PackageReference Include="MongoDB.Bson" Version="2.22.0" />
+    <PackageReference Include="MongoDB.Bson" Version="2.13.2" />
   </ItemGroup>
 
-</Project>
+</Project>

Streetwriters.Data/UnitOfWork.cs

@@ -23,16 +23,24 @@ using Streetwriters.Data.Interfaces;
 
 namespace Streetwriters.Data
 {
-    public class UnitOfWork(IDbContext dbContext) : IUnitOfWork
+    public class UnitOfWork : IUnitOfWork
     {
+        private readonly IDbContext dbContext;
+
+        public UnitOfWork(IDbContext _dbContext)
+        {
+            dbContext = _dbContext;
+        }
+
         public async Task<bool> Commit()
         {
-            return await dbContext.SaveChanges() > 0;
+            var changeAmount = await dbContext.SaveChanges();
+            return changeAmount > 0;
         }
 
         public void Dispose()
         {
-            dbContext.Dispose();
+            this.dbContext.Dispose();
         }
     }
 }

Streetwriters.Identity/Config.cs

@@ -20,7 +20,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 using IdentityServer4;
 using IdentityServer4.Models;
 using Streetwriters.Common;
+using System;
 using System.Collections.Generic;
+using System.Linq;
 
 namespace Streetwriters.Identity
 {
@@ -76,8 +78,8 @@ namespace Streetwriters.Identity
                     RefreshTokenUsage = TokenUsage.ReUse,
                     RefreshTokenExpiration = TokenExpiration.Sliding,
 
-                    AccessTokenLifetime = 6 * 3600, // 6 hours
+                    AccessTokenLifetime = 3600, // 1 hour
-                    SlidingRefreshTokenLifetime = 45 * 3600 * 24, // 45 days
+                    SlidingRefreshTokenLifetime = 15 * 60 * 60 * 24, // 15 days
                     AbsoluteRefreshTokenLifetime = 0, // 0 means infinite sliding lifetime
                     
                     // scopes that client has access to

Streetwriters.Identity/Controllers/AccountController.cs

@@ -21,25 +21,19 @@ using System.Collections.Generic;
 using System.ComponentModel;
 using System.Linq;
 using System.Security.Claims;
-using System.Text.Json;
 using System.Threading.Tasks;
 using AspNetCore.Identity.Mongo.Model;
-using IdentityServer4;
 using IdentityServer4.Configuration;
-using IdentityServer4.Models;
 using IdentityServer4.Stores;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Mvc;
 using Streetwriters.Common;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Common.Models;
 using Streetwriters.Identity.Enums;
 using Streetwriters.Identity.Interfaces;
 using Streetwriters.Identity.Models;
-using Streetwriters.Identity.Services;
 using static IdentityServer4.IdentityServerConstants;
 
 namespace Streetwriters.Identity.Controllers
@@ -54,14 +48,12 @@ namespace Streetwriters.Identity.Controllers
         private ITokenGenerationService TokenGenerationService { get; set; }
         private IUserClaimsPrincipalFactory<User> PrincipalFactory { get; set; }
         private IdentityServerOptions ISOptions { get; set; }
-        private IUserAccountService UserAccountService { get; set; }
         public AccountController(UserManager<User> _userManager, IEmailSender _emailSender,
         SignInManager<User> _signInManager, RoleManager<MongoRole> _roleManager, IPersistedGrantStore store,
-        ITokenGenerationService tokenGenerationService, IMFAService _mfaService, IUserAccountService userAccountService) : base(_userManager, _emailSender, _signInManager, _roleManager, _mfaService)
+        ITokenGenerationService tokenGenerationService, IMFAService _mfaService) : base(_userManager, _emailSender, _signInManager, _roleManager, _mfaService)
         {
             PersistedGrantStore = store;
             TokenGenerationService = tokenGenerationService;
-            UserAccountService = userAccountService;
         }
 
         [HttpGet("confirm")]
@@ -73,7 +65,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.FindByIdAsync(userId);
-            if (!await UserService.IsUserValidAsync(UserManager, user, clientId)) return BadRequest($"Unable to find user with ID '{userId}'.");
+            if (!await IsUserValidAsync(user, clientId)) return BadRequest($"Unable to find user with ID '{userId}'.");
 
             switch (type)
             {
@@ -84,20 +76,30 @@ namespace Streetwriters.Identity.Controllers
                         var result = await UserManager.ConfirmEmailAsync(user, code);
                         if (!result.Succeeded) return BadRequest(result.Errors.ToErrors());
 
+
                         if (await UserManager.IsInRoleAsync(user, client.Id))
                         {
                             await client.OnEmailConfirmed(userId);
-                        }
+                            // if (client.WelcomeEmailTemplateId != null)
-
+                            //     await EmailSender.SendWelcomeEmailAsync(user.Email, client);
-                        if (!await UserManager.GetTwoFactorEnabledAsync(user))
-                        {
-                            await MFAService.EnableMFAAsync(user, MFAMethods.Email);
-                            user = await UserManager.GetUserAsync(User);
                         }
 
                         var redirectUrl = $"{client.EmailConfirmedRedirectURL}?userId={userId}";
                         return RedirectPermanent(redirectUrl);
                     }
+                // case TokenType.CHANGE_EMAIL:
+                //     {
+                //         var newEmail = user.Claims.Find((c) => c.ClaimType == "new_email");
+                //         if (newEmail == null) return BadRequest("Email change was not requested.");
+
+                //         var result = await UserManager.ChangeEmailAsync(user, newEmail.ClaimValue.ToString(), code);
+                //         if (result.Succeeded)
+                //         {
+                //             await UserManager.RemoveClaimAsync(user, newEmail.ToClaim());
+                //             return Ok("Email changed.");
+                //         }
+                //         return BadRequest("Could not change email.");
+                //     }
                 case TokenType.RESET_PASSWORD:
                     {
                         if (!await UserManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword", code))
@@ -120,7 +122,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
 
             if (string.IsNullOrEmpty(newEmail))
             {
@@ -136,13 +138,51 @@ namespace Streetwriters.Identity.Controllers
             return Ok();
         }
 
+        [HttpPost("unregister")]
+        public async Task<IActionResult> UnregisterAccountAync([FromForm] DeleteAccountForm form)
+        {
+            var client = Clients.FindClientById(User.FindFirstValue("client_id"));
+            if (client == null) return BadRequest("Invalid client_id.");
+
+            var user = await UserManager.GetUserAsync(User);
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+
+            if (!await UserManager.CheckPasswordAsync(user, form.Password))
+            {
+                return Unauthorized();
+            }
+
+            await UserManager.RemoveFromRoleAsync(user, client.Id);
+
+            IdentityUserClaim<string> statusClaim = user.Claims.FirstOrDefault((c) => c.ClaimType == $"{client.Id}:status");
+            await UserManager.RemoveClaimAsync(user, statusClaim.ToClaim());
+            return Ok();
+        }
+
         [HttpGet]
         public async Task<IActionResult> GetUserAccount()
         {
             var client = Clients.FindClientById(User.FindFirstValue("client_id"));
             if (client == null) return BadRequest("Invalid client_id.");
+
             var user = await UserManager.GetUserAsync(User);
-            return Ok(UserAccountService.GetUserAsync(client.Id, user.Id.ToString()));
+            if (!await IsUserValidAsync(user, client.Id))
+                return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+
+            return Ok(new UserModel
+            {
+                UserId = user.Id.ToString(),
+                Email = user.Email,
+                IsEmailConfirmed = user.EmailConfirmed,
+                // PhoneNumber = user.PhoneNumberConfirmed ? user.PhoneNumber : null,
+                MFA = new MFAConfig
+                {
+                    IsEnabled = user.TwoFactorEnabled,
+                    PrimaryMethod = MFAService.GetPrimaryMethod(user),
+                    SecondaryMethod = MFAService.GetSecondaryMethod(user),
+                    RemainingValidCodes = await MFAService.GetRemainingValidCodesAsync(user)
+                }
+            });
         }
 
         [HttpPost("recover")]
@@ -153,7 +193,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.FindByEmailAsync(form.Email);
-            if (!await UserService.IsUserValidAsync(UserManager, user, form.ClientId)) return Ok();
+            if (!await IsUserValidAsync(user, form.ClientId)) return Ok();
 
             var code = await UserManager.GenerateUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword");
             var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.RESET_PASSWORD, Request.Scheme);
@@ -173,7 +213,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
 
             var subjectId = User.FindFirstValue("sub");
             var jti = User.FindFirstValue("jti");
@@ -200,7 +240,7 @@ namespace Streetwriters.Identity.Controllers
         {
             if (!Clients.IsValidClient(form.ClientId)) return BadRequest("Invalid clientId.");
             var user = await UserManager.FindByIdAsync(form.UserId);
-            if (!await UserService.IsUserValidAsync(UserManager, user, form.ClientId))
+            if (!await IsUserValidAsync(user, form.ClientId))
                 return BadRequest($"Unable to find user with ID '{form.UserId}'.");
 
             if (!await UserManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, "PasswordResetAuthorizationCode", form.Code))
@@ -209,7 +249,6 @@ namespace Streetwriters.Identity.Controllers
             return Ok(new
             {
                 access_token = token,
-                scope = string.Join(' ', Config.ApiScopes.Select(s => s.Name)),
                 expires_in = 18000
             });
         }
@@ -221,7 +260,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id))
+            if (!await IsUserValidAsync(user, client.Id))
                 return BadRequest($"Unable to find user with ID '{UserManager.GetUserId(User)}'.");
 
             switch (form.Type)
@@ -238,7 +277,7 @@ namespace Streetwriters.Identity.Controllers
                                 if (result.Succeeded)
                                 {
                                     await UserManager.SetUserNameAsync(user, form.NewEmail);
-                                    await SendLogoutMessageAsync(user.Id.ToString(), "Email changed.");
+                                    await SendEmailChangedMessageAsync(user.Id.ToString());
                                     return Ok();
                                 }
                             }
@@ -250,7 +289,7 @@ namespace Streetwriters.Identity.Controllers
                         var result = await UserManager.ChangePasswordAsync(user, form.OldPassword, form.NewPassword);
                         if (result.Succeeded)
                         {
-                            await SendLogoutMessageAsync(user.Id.ToString(), "Password changed.");
+                            await SendPasswordChangedMessageAsync(user.Id.ToString());
                             return Ok();
                         }
                         return BadRequest(result.Errors.ToErrors());
@@ -260,27 +299,15 @@ namespace Streetwriters.Identity.Controllers
                         var result = await UserManager.RemovePasswordAsync(user);
                         if (result.Succeeded)
                         {
-                            await MFAService.ResetMFAAsync(user);
                             result = await UserManager.AddPasswordAsync(user, form.NewPassword);
                             if (result.Succeeded)
                             {
-                                await SendLogoutMessageAsync(user.Id.ToString(), "Password reset.");
+                                await SendPasswordChangedMessageAsync(user.Id.ToString());
                                 return Ok();
                             }
                         }
                         return BadRequest(result.Errors.ToErrors());
                     }
-                case "change_marketing_consent":
-                    {
-                        var claimType = $"{client.Id}:marketing_consent";
-                        var claims = await UserManager.GetClaimsAsync(user);
-                        var marketingConsentClaim = claims.FirstOrDefault((claim) => claim.Type == claimType);
-                        if (marketingConsentClaim != null) await UserManager.RemoveClaimAsync(user, marketingConsentClaim);
-                        if (!form.Enabled)
-                            await UserManager.AddClaimAsync(user, new Claim(claimType, "false"));
-                        return Ok();
-                    }
-
             }
             return BadRequest("Invalid type.");
         }
@@ -292,7 +319,7 @@ namespace Streetwriters.Identity.Controllers
             if (client == null) return BadRequest("Invalid client_id.");
 
             var user = await UserManager.GetUserAsync(User);
-            if (!await UserService.IsUserValidAsync(UserManager, user, client.Id)) return BadRequest($"Unable to find user with ID '{user.Id}'.");
+            if (!await IsUserValidAsync(user, client.Id)) return BadRequest($"Unable to find user with ID '{user.Id.ToString()}'.");
 
             var jti = User.FindFirstValue("jti");
 
@@ -301,44 +328,43 @@ namespace Streetwriters.Identity.Controllers
                 ClientId = client.Id,
                 SubjectId = user.Id.ToString()
             });
-            var refreshTokenKey = GetHashedKey(refresh_token, PersistedGrantTypes.RefreshToken);
-            var removedKeys = new List<string>();
             foreach (var grant in grants)
             {
-                if (!all && (grant.Data.Contains(jti) || grant.Key == refreshTokenKey)) continue;
+                if (!all && (grant.Data.Contains(jti) || grant.Data.Contains(refresh_token))) continue;
                 await PersistedGrantStore.RemoveAsync(grant.Key);
-                removedKeys.Add(grant.Key);
             }
-
-            await WampServers.NotesnookServer.PublishMessageAsync(IdentityServerTopics.ClearCacheTopic, new ClearCacheMessage(removedKeys));
-            await WampServers.MessengerServer.PublishMessageAsync(IdentityServerTopics.ClearCacheTopic, new ClearCacheMessage(removedKeys));
-            await WampServers.SubscriptionServer.PublishMessageAsync(IdentityServerTopics.ClearCacheTopic, new ClearCacheMessage(removedKeys));
-            await SendLogoutMessageAsync(user.Id.ToString(), "Session revoked.");
             return Ok();
         }
 
-        private static string GetHashedKey(string value, string grantType)
+        private async Task SendPasswordChangedMessageAsync(string userId)
         {
-            return (value + ":" + grantType).Sha256();
+            await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
-        }
-
-        private async Task SendLogoutMessageAsync(string userId, string reason)
-        {
-            await SendMessageAsync(userId, new Message
-            {
-                Type = "logout",
-                Data = JsonSerializer.Serialize(new { reason })
-            });
-        }
-
-        private async Task SendMessageAsync(string userId, Message message)
-        {
-            await WampServers.MessengerServer.PublishMessageAsync(MessengerServerTopics.SendSSETopic, new SendSSEMessage
             {
                 UserId = userId,
                 OriginTokenId = User.FindFirstValue("jti"),
-                Message = message
+                Message = new Message
+                {
+                    Type = "userPasswordChanged"
+                }
             });
         }
+
+        private async Task SendEmailChangedMessageAsync(string userId)
+        {
+            await WampServers.MessengerServer.PublishMessageAsync(WampServers.MessengerServer.Topics.SendSSETopic, new SendSSEMessage
+            {
+                UserId = userId,
+                OriginTokenId = User.FindFirstValue("jti"),
+                Message = new Message
+                {
+                    Type = "userEmailChanged"
+                }
+            });
+        }
+
+        public async Task<bool> IsUserValidAsync(User user, string clientId)
+        {
+            return user != null && await UserManager.IsInRoleAsync(user, clientId);
+        }
     }
 }

Streetwriters.Identity/Controllers/MFAController.cs

@@ -74,9 +74,21 @@ namespace Streetwriters.Identity.Controllers
         }
 
         [HttpDelete]
-        public IActionResult Disable2FA()
+        public async Task<IActionResult> Disable2FA()
         {
-            return BadRequest("2FA is mandatory and cannot be disabled.");
+            var user = await UserManager.GetUserAsync(User);
+
+            if (!await UserManager.GetTwoFactorEnabledAsync(user))
+            {
+                return BadRequest("Cannot disable 2FA as it's not currently enabled");
+            }
+
+            if (await MFAService.DisableMFAAsync(user))
+            {
+                return Ok();
+            }
+
+            return BadRequest("Failed to disable 2FA.");
         }
 
         [HttpGet("codes")]

Streetwriters.Identity/Controllers/SignupController.cs

@@ -17,6 +17,7 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
@@ -26,7 +27,6 @@ using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.AspNetCore.Mvc;
 using Streetwriters.Common;
-using Streetwriters.Common.Enums;
 using Streetwriters.Common.Models;
 using Streetwriters.Identity.Enums;
 using Streetwriters.Identity.Interfaces;
@@ -53,87 +53,68 @@ namespace Streetwriters.Identity.Controllers
         [AllowAnonymous]
         public async Task<IActionResult> Signup([FromForm] SignupForm form)
         {
-            if (Constants.DISABLE_ACCOUNT_CREATION)
+            var client = Clients.FindClientById(form.ClientId);
-                return BadRequest(new string[] { "Creating new accounts is not allowed." });
+            if (client == null) return BadRequest(new string[] { "Invalid client id." });
-            try
+
+            await AddClientRoleAsync(client.Id);
+
+            // email addresses must be case-insensitive
+            form.Email = form.Email.ToLowerInvariant();
+            form.Username = form.Username?.ToLowerInvariant();
+
+            if (!await EmailAddressValidator.IsEmailAddressValidAsync(form.Email)) return BadRequest(new string[] { "Invalid email address." });
+
+            var result = await UserManager.CreateAsync(new User
             {
-                var client = Clients.FindClientById(form.ClientId);
+                Email = form.Email,
-                if (client == null) return BadRequest(new string[] { "Invalid client id." });
+                EmailConfirmed = false,
+                UserName = form.Username ?? form.Email,
+            }, form.Password);
 
-                await AddClientRoleAsync(client.Id);
+            if (result.Errors.Any((e) => e.Code == "DuplicateEmail"))
+            {
+                var user = await UserManager.FindByEmailAsync(form.Email);
 
-                // email addresses must be case-insensitive
+                if (!await UserManager.IsInRoleAsync(user, client.Id))
-                form.Email = form.Email.ToLowerInvariant();
-                form.Username = form.Username?.ToLowerInvariant();
-
-                if (!await EmailAddressValidator.IsEmailAddressValidAsync(form.Email)) return BadRequest(new string[] { "Invalid email address." });
-
-                var result = await UserManager.CreateAsync(new User
                 {
-                    Email = form.Email,
+                    if (!await UserManager.CheckPasswordAsync(user, form.Password))
-                    EmailConfirmed = Constants.IS_SELF_HOSTED,
-                    UserName = form.Username ?? form.Email,
-                }, form.Password);
-
-                if (result.Errors.Any((e) => e.Code == "DuplicateEmail"))
-                {
-                    var user = await UserManager.FindByEmailAsync(form.Email);
-
-                    if (!await UserManager.IsInRoleAsync(user, client.Id))
                     {
-                        if (!await UserManager.CheckPasswordAsync(user, form.Password))
+                        // TODO
-                        {
+                        await UserManager.RemovePasswordAsync(user);
-                            // TODO
+                        await UserManager.AddPasswordAsync(user, form.Password);
-                            await UserManager.RemovePasswordAsync(user);
-                            await UserManager.AddPasswordAsync(user, form.Password);
-                        }
-                        await MFAService.DisableMFAAsync(user);
-                        await UserManager.AddToRoleAsync(user, client.Id);
                     }
-                    else
-                    {
-                        return BadRequest(new string[] { "Invalid email address.." });
-                    }
-
-                    return Ok(new
-                    {
-                        userId = user.Id.ToString()
-                    });
-                }
-
-                if (result.Succeeded)
-                {
-                    var user = await UserManager.FindByEmailAsync(form.Email);
                     await UserManager.AddToRoleAsync(user, client.Id);
-                    if (Constants.IS_SELF_HOSTED)
+                }
-                    {
+                else
-                        await UserManager.AddClaimAsync(user, UserService.SubscriptionTypeToClaim(client.Id, Common.Enums.SubscriptionType.PREMIUM));
+                {
-                    }
+                    return BadRequest(new string[] { "Invalid email address." });
-                    else
-                    {
-                        await UserManager.AddClaimAsync(user, new Claim("platform", PlatformFromUserAgent(base.HttpContext.Request.Headers.UserAgent)));
-                        var code = await UserManager.GenerateEmailConfirmationTokenAsync(user);
-                        var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.CONFRIM_EMAIL, Request.Scheme);
-                        await EmailSender.SendConfirmationEmailAsync(user.Email, callbackUrl, client);
-                    }
-                    return Ok(new
-                    {
-                        userId = user.Id.ToString()
-                    });
                 }
 
-                return BadRequest(result.Errors.ToErrors());
+                return Ok(new
+                {
+                    userId = user.Id.ToString()
+                });
             }
-            catch (System.Exception ex)
-            {
-                await Slogger<SignupController>.Error("Signup", ex.ToString());
-                return BadRequest("Failed to create an account.");
-            }
-        }
 
-        string PlatformFromUserAgent(string userAgent)
+            if (result.Succeeded)
-        {
+            {
-            return userAgent.Contains("okhttp/") ? "android" : userAgent.Contains("Darwin/") || userAgent.Contains("CFNetwork/") ? "ios" : "web";
+                var user = await UserManager.FindByEmailAsync(form.Email);
+
+                await UserManager.AddToRoleAsync(user, client.Id);
+                if (Constants.IS_SELF_HOSTED)
+                    await UserManager.AddClaimAsync(user, UserService.SubscriptionTypeToClaim(client.Id, Common.Enums.SubscriptionType.PREMIUM));
+
+                var code = await UserManager.GenerateEmailConfirmationTokenAsync(user);
+                var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.CONFRIM_EMAIL, Request.Scheme);
+                await EmailSender.SendConfirmationEmailAsync(user.Email, callbackUrl, client);
+
+                return Ok(new
+                {
+                    userId = user.Id.ToString()
+                });
+            }
+
+            return BadRequest(result.Errors.ToErrors());
         }
     }
 }

Streetwriters.Identity/Dockerfile

@@ -1,52 +1,28 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build
 WORKDIR /app
-EXPOSE 80
-EXPOSE 443
-
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
-ARG TARGETARCH
-ARG BUILDPLATFORM
-ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
-
-WORKDIR /src
 
+# restore all project dependencies
 COPY Streetwriters.Data/*.csproj ./Streetwriters.Data/
+RUN dotnet restore /app/Streetwriters.Data/Streetwriters.Data.csproj --use-current-runtime
+
 COPY Streetwriters.Common/*.csproj ./Streetwriters.Common/
+RUN dotnet restore /app/Streetwriters.Common/Streetwriters.Common.csproj --use-current-runtime
+
 COPY Streetwriters.Identity/*.csproj ./Streetwriters.Identity/
+RUN dotnet restore /app/Streetwriters.Identity/Streetwriters.Identity.csproj --use-current-runtime
 
-# restore dependencies
+# copy everything else
-RUN dotnet restore -v d /src/Streetwriters.Identity/Streetwriters.Identity.csproj --use-current-runtime
-
 COPY Streetwriters.Data/ ./Streetwriters.Data/
 COPY Streetwriters.Common/ ./Streetwriters.Common/
 COPY Streetwriters.Identity/ ./Streetwriters.Identity/
 
-WORKDIR /src/Streetwriters.Identity/
+# build
+WORKDIR /app/Streetwriters.Identity/
+ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
+RUN dotnet publish -c Release -o /app/out --use-current-runtime --self-contained false --no-restore
 
-RUN dotnet build -c Release -o /app/build -a $TARGETARCH
+# final stage/image
-
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
-FROM build AS publish
-RUN dotnet publish -c Release -o /app/publish \
-  #--runtime alpine-x64 \
-  --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
-  /p:PublishSingleFile=true \
-  /p:JsonSerializerIsReflectionEnabledByDefault=true \
-  -a $TARGETARCH
-
-FROM --platform=$BUILDPLATFORM base AS final
-ARG TARGETARCH
-ARG BUILDPLATFORM
-
-# create a new user and change directory ownership
-RUN adduser --disabled-password \
-  --home /app \
-  --gecos '' dotnetuser && chown -R dotnetuser /app
-
-# impersonate into the new user
-USER dotnetuser
 WORKDIR /app
-
+COPY --from=build /app/out .
-COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "Streetwriters.Identity.dll"]
-ENTRYPOINT ["./Streetwriters.Identity"]

Streetwriters.Identity/Interfaces/IMFAService.cs

@@ -28,7 +28,6 @@ namespace Streetwriters.Identity.Interfaces
     {
         Task EnableMFAAsync(User user, string primaryMethod);
         Task<bool> DisableMFAAsync(User user);
-        Task<bool> ResetMFAAsync(User user);
         Task SetSecondaryMethodAsync(User user, string secondaryMethod);
         string GetPrimaryMethod(User user);
         string GetSecondaryMethod(User user);

Streetwriters.Identity/Interfaces/ISMSSender.cs

@@ -24,7 +24,7 @@ namespace Streetwriters.Identity.Interfaces
 {
     public interface ISMSSender
     {
-        Task<string> SendOTPAsync(string number, IClient client);
+        string SendOTP(string number, IClient client);
-        Task<bool> VerifyOTPAsync(string id, string code);
+        bool VerifyOTP(string id, string code);
     }
 }

Streetwriters.Identity/MessageHandlers/CreateSubscription.cs

@@ -36,7 +36,7 @@ namespace Streetwriters.Identity.MessageHandlers
             var client = Clients.FindClientByAppId(message.AppId);
             if (client == null || user == null) return;
 
-            IdentityUserClaim<string> statusClaim = user.Claims.FirstOrDefault((c) => c.ClaimType == UserService.GetClaimKey(client.Id));
+            IdentityUserClaim<string> statusClaim = user.Claims.FirstOrDefault((c) => c.ClaimType == $"{client.Id}:status");
             Claim subscriptionClaim = UserService.SubscriptionTypeToClaim(client.Id, message.Type);
             if (statusClaim?.ClaimValue == subscriptionClaim.Value) return;
             if (statusClaim != null)

Streetwriters.Identity/Models/DeleteAccountForm.cs

@@ -17,22 +17,17 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 using System.Runtime.Serialization;
-using System.Text.Json.Serialization;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
 
-namespace Streetwriters.Common.Messages
+namespace Streetwriters.Identity.Models
 {
-    public class ClearCacheMessage
+    public class DeleteAccountForm
     {
-        public ClearCacheMessage(List<string> keys)
+        [Required]
+        public string Password
         {
-            this.Keys = keys;
+            get; set;
         }
-
-        [JsonPropertyName("keys")]
-        public List<string> Keys { get; set; }
     }
 }

Streetwriters.Identity/Models/UpdateUserForm.cs

@@ -32,12 +32,6 @@ namespace Streetwriters.Identity.Models
             get; set;
         }
 
-        [BindProperty(Name = "enabled")]
-        public bool Enabled
-        {
-            get; set;
-        }
-
         [BindProperty(Name = "old_password")]
         public string OldPassword
         {

Streetwriters.Identity/Services/CustomIntrospectionResponseGenerator.cs

@@ -46,7 +46,6 @@ namespace Streetwriters.Identity.Services
             if (result.TryGetValue("sub", out object userId))
             {
                 var user = await UserManager.FindByIdAsync(userId.ToString());
-                if (user == null || user.Claims == null) return result;
 
                 var verifiedClaim = user.Claims.Find((c) => c.ClaimType == "verified");
                 if (verifiedClaim != null)

Streetwriters.Identity/Services/CustomPersistedGrantDbContext.cs

@@ -67,7 +67,7 @@ namespace Streetwriters.Identity.Services
 
         public Task RemoveExpired()
         {
-            return Remove(x => x.Type == "reference_token" && x.Expiration.HasValue && x.Expiration.Value < DateTime.UtcNow);
+            return Remove(x => x.Expiration < DateTime.UtcNow.AddHours(12));
         }
 
         public Task InsertOrUpdate(Expression<Func<PersistedGrant, bool>> filter, PersistedGrant entity)

Streetwriters.Identity/Services/EmailAddressValidator.cs

@@ -3,14 +3,13 @@ using System.Collections.Generic;
 using System.Net.Http;
 using System.Threading.Tasks;
 using Streetwriters.Common;
-using System.Linq;
 
 namespace Streetwriters.Identity.Services
 {
     public class EmailAddressValidator
     {
         private static DateTimeOffset LAST_FETCH_TIME = DateTimeOffset.MinValue;
-        private static HashSet<string> BLACKLISTED_DOMAINS = new();
+        private static HashSet<string> BLACKLISTED_DOMAINS = new HashSet<string>();
 
         public static async Task<bool> IsEmailAddressValidAsync(string email)
         {
@@ -20,9 +19,8 @@ namespace Streetwriters.Identity.Services
                 if (LAST_FETCH_TIME.AddDays(1) < DateTimeOffset.UtcNow)
                 {
                     var httpClient = new HttpClient();
-                    var domainsList = await httpClient.GetStringAsync("https://raw.githubusercontent.com/disposable-email-domains/disposable-email-domains/master/disposable_email_blocklist.conf");
+                    var domainsList = await httpClient.GetStringAsync("https://disposable.github.io/disposable-email-domains/domains.txt");
-                    var domains = domainsList.Split('\n').Where(line => !string.IsNullOrWhiteSpace(line) && !line.TrimStart().StartsWith("//"));
+                    BLACKLISTED_DOMAINS = new HashSet<string>(domainsList.Split('\n'));
-                    BLACKLISTED_DOMAINS = new HashSet<string>(domains, StringComparer.OrdinalIgnoreCase);
                     LAST_FETCH_TIME = DateTimeOffset.UtcNow;
                 }
 

Streetwriters.Identity/Services/EmailSender.cs

@@ -231,9 +231,8 @@ namespace Streetwriters.Identity.Services
                     return builder.ToMessageBody();
                 }
             }
-            catch (Exception ex)
+            catch (PrivateKeyNotFoundException)
             {
-                await Slogger<EmailSender>.Error("GetEmailBodyAsync", ex.ToString());
                 return builder.ToMessageBody();
             }
         }

Streetwriters.Identity/Services/MFAService.cs

@@ -54,7 +54,6 @@ namespace Streetwriters.Identity.Services
             if (!result.Succeeded) return;
 
             await this.RemovePrimaryMethodAsync(user);
-            await this.RemoveSecondaryMethodAsync(user);
             await UserManager.AddClaimAsync(user, new Claim(MFAService.PRIMARY_METHOD_CLAIM, primaryMethod));
         }
 
@@ -70,20 +69,6 @@ namespace Streetwriters.Identity.Services
             return true;
         }
 
-        public async Task<bool> ResetMFAAsync(User user)
-        {
-            await UserManager.SetTwoFactorEnabledAsync(user, false);
-            await UserManager.SetTwoFactorEnabledAsync(user, true);
-
-            await this.RemovePrimaryMethodAsync(user);
-            await this.RemoveSecondaryMethodAsync(user);
-
-            await UserManager.AddClaimAsync(user, new Claim(MFAService.PRIMARY_METHOD_CLAIM, MFAMethods.Email));
-
-            await UserManager.ResetAuthenticatorKeyAsync(user);
-            return true;
-        }
-
         public async Task SetSecondaryMethodAsync(User user, string secondaryMethod)
         {
             await this.ReplaceClaimAsync(user, MFAService.SECONDARY_METHOD_CLAIM, secondaryMethod);
@@ -97,7 +82,7 @@ namespace Streetwriters.Identity.Services
 
         public string GetPrimaryMethod(User user)
         {
-            return this.GetClaimValue(user, MFAService.PRIMARY_METHOD_CLAIM, MFAMethods.Email);
+            return this.GetClaimValue(user, MFAService.PRIMARY_METHOD_CLAIM);
         }
 
         public string GetSecondaryMethod(User user)
@@ -105,10 +90,10 @@ namespace Streetwriters.Identity.Services
             return this.GetClaimValue(user, MFAService.SECONDARY_METHOD_CLAIM);
         }
 
-        public string GetClaimValue(User user, string claimType, string defaultValue = null)
+        public string GetClaimValue(User user, string claimType)
         {
             var claim = user.Claims.FirstOrDefault((c) => c.ClaimType == claimType);
-            return claim != null ? claim.ClaimValue : defaultValue;
+            return claim != null ? claim.ClaimValue : null;
         }
 
         public Task<int> GetRemainingValidCodesAsync(User user)
@@ -176,7 +161,7 @@ namespace Streetwriters.Identity.Services
                     break;
                 case "sms":
                     await UserManager.SetPhoneNumberAsync(user, form.PhoneNumber);
-                    var id = await SMSSender.SendOTPAsync(form.PhoneNumber, client);
+                    var id = SMSSender.SendOTP(form.PhoneNumber, client);
                     await this.ReplaceClaimAsync(user, MFAService.SMS_ID_CLAIM, id);
                     break;
 
@@ -189,7 +174,7 @@ namespace Streetwriters.Identity.Services
             {
                 var id = this.GetClaimValue(user, MFAService.SMS_ID_CLAIM);
                 if (string.IsNullOrEmpty(id)) throw new Exception("Could not find associated SMS verify id. Please try sending the code again.");
-                if (await SMSSender.VerifyOTPAsync(id, code))
+                if (SMSSender.VerifyOTP(id, code))
                 {
                     // Auto confirm user phone number if not confirmed
                     if (!await UserManager.IsPhoneNumberConfirmedAsync(user))

Streetwriters.Identity/Services/SMSSender.cs

@@ -24,10 +24,6 @@ using MessageBird.Objects;
 using Microsoft.Extensions.Options;
 using Streetwriters.Identity.Models;
 using Streetwriters.Common;
-using Twilio.Rest.Verify.V2.Service;
-using Twilio;
-using System.Threading.Tasks;
-using System;
 
 namespace Streetwriters.Identity.Services
 {
@@ -36,29 +32,30 @@ namespace Streetwriters.Identity.Services
         private Client client;
         public SMSSender()
         {
-            if (!string.IsNullOrEmpty(Constants.TWILIO_ACCOUNT_SID) && !string.IsNullOrEmpty(Constants.TWILIO_AUTH_TOKEN))
+            if (!string.IsNullOrEmpty(Constants.MESSAGEBIRD_ACCESS_KEY))
+                client = Client.CreateDefault(Constants.MESSAGEBIRD_ACCESS_KEY);
+        }
+
+        public string SendOTP(string number, IClient app)
+        {
+            VerifyOptionalArguments optionalArguments = new VerifyOptionalArguments
             {
-                TwilioClient.Init(Constants.TWILIO_ACCOUNT_SID, Constants.TWILIO_AUTH_TOKEN);
+                Originator = app.Name,
-            }
+                Reference = app.Name,
+                Type = MessageType.Sms,
+                Template = $"Your {app.Name} 2FA code is: %token. Valid for 5 minutes.",
+                TokenLength = 6,
+                Timeout = 60 * 5
+            };
+            Verify verify = client.CreateVerify(number, optionalArguments);
+            if (verify.Status == VerifyStatus.Sent) return verify.Id;
+            return null;
         }
 
-        public async Task<string> SendOTPAsync(string number, IClient app)
+        public bool VerifyOTP(string id, string code)
         {
-            var verification = await VerificationResource.CreateAsync(
+            Verify verify = client.SendVerifyToken(id, code);
-                to: number,
+            return verify.Status == VerifyStatus.Verified;
-                channel: "sms",
-                pathServiceSid: Constants.TWILIO_SERVICE_SID
-            );
-            return verification.Sid;
-        }
-
-        public async Task<bool> VerifyOTPAsync(string id, string code)
-        {
-            return (await VerificationCheckResource.CreateAsync(
-                verificationSid: id,
-                pathServiceSid: Constants.TWILIO_SERVICE_SID,
-                code: code
-            )).Status == "approved";
         }
     }
 }

Streetwriters.Identity/Services/TokenGenerationService.cs

@@ -84,13 +84,11 @@ namespace Streetwriters.Identity.Helpers
         public async Task<ClaimsPrincipal> TransformTokenRequestAsync(ValidatedTokenRequest request, User user, string grantType, string[] scopes, int lifetime = 20 * 60)
         {
             var principal = await PrincipalFactory.CreateAsync(user);
-            var identityUser = new IdentityServerUser(user.Id.ToString())
+            var identityUser = new IdentityServerUser(user.Id.ToString());
-            {
+            identityUser.DisplayName = user.UserName;
-                DisplayName = user.UserName,
+            identityUser.AuthenticationTime = System.DateTime.UtcNow;
-                AuthenticationTime = System.DateTime.UtcNow,
+            identityUser.IdentityProvider = IdentityServerConstants.LocalIdentityProvider;
-                IdentityProvider = IdentityServerConstants.LocalIdentityProvider,
+            identityUser.AdditionalClaims = principal.Claims.ToArray();
-                AdditionalClaims = principal.Claims.ToArray()
-            };
 
             request.AccessTokenType = AccessTokenType.Jwt;
             request.AccessTokenLifetime = lifetime;

Streetwriters.Identity/Services/UserAccountService.cs

@@ -1,56 +0,0 @@
-using System;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Identity;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
-using Streetwriters.Identity.Interfaces;
-using Streetwriters.Identity.Models;
-
-namespace Streetwriters.Identity.Services
-{
-    public class UserAccountService(UserManager<User> userManager, IMFAService mfaService) : IUserAccountService
-    {
-        public async Task<UserModel> GetUserAsync(string clientId, string userId)
-        {
-            var user = await userManager.FindByIdAsync(userId);
-            if (!await UserService.IsUserValidAsync(userManager, user, clientId))
-                throw new Exception($"Unable to find user with ID '{userId}'.");
-
-            var claims = await userManager.GetClaimsAsync(user);
-            var marketingConsentClaim = claims.FirstOrDefault((claim) => claim.Type == $"{clientId}:marketing_consent");
-
-            if (await userManager.IsEmailConfirmedAsync(user) && !await userManager.GetTwoFactorEnabledAsync(user))
-            {
-                await mfaService.EnableMFAAsync(user, MFAMethods.Email);
-                user = await userManager.FindByIdAsync(userId);
-            }
-
-            return new UserModel
-            {
-                UserId = user.Id.ToString(),
-                Email = user.Email,
-                IsEmailConfirmed = user.EmailConfirmed,
-                MarketingConsent = marketingConsentClaim == null,
-                MFA = new MFAConfig
-                {
-                    IsEnabled = user.TwoFactorEnabled,
-                    PrimaryMethod = mfaService.GetPrimaryMethod(user),
-                    SecondaryMethod = mfaService.GetSecondaryMethod(user),
-                    RemainingValidCodes = await mfaService.GetRemainingValidCodesAsync(user)
-                }
-            };
-        }
-
-        public async Task DeleteUserAsync(string clientId, string userId, string password)
-        {
-            var user = await userManager.FindByIdAsync(userId);
-            if (!await UserService.IsUserValidAsync(userManager, user, clientId)) throw new Exception($"User not found.");
-
-            if (!await userManager.CheckPasswordAsync(user, password)) throw new Exception("Wrong password.");
-
-            await userManager.DeleteAsync(user);
-        }
-    }
-}

Streetwriters.Identity/Services/UserService.cs

@@ -19,8 +19,6 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System.Linq;
 using System.Security.Claims;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Identity;
 using Streetwriters.Common.Enums;
 using Streetwriters.Common.Models;
 
@@ -80,10 +78,5 @@ namespace Streetwriters.Identity.Services
         {
             return $"{clientId}:status";
         }
-
-        public static async Task<bool> IsUserValidAsync(UserManager<User> userManager, User user, string clientId)
-        {
-            return user != null && await userManager.IsInRoleAsync(user, clientId);
-        }
     }
 }

Streetwriters.Identity/Startup.cs

@@ -40,7 +40,6 @@ using MongoDB.Bson.Serialization;
 using Quartz;
 using Streetwriters.Common;
 using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
 using Streetwriters.Common.Models;
 using Streetwriters.Identity.Helpers;
@@ -166,7 +165,6 @@ namespace Streetwriters.Identity
 
             AddOperationalStore(services, new TokenCleanupOptions { Enable = true, Interval = 3600 * 12 });
 
-            services.AddScoped<IUserAccountService, UserAccountService>();
             services.AddTransient<IMFAService, MFAService>();
             services.AddControllers();
             services.AddTransient<IIntrospectionResponseGenerator, CustomIntrospectionResponseGenerator>();
@@ -192,7 +190,7 @@ namespace Streetwriters.Identity
             }
 
             app.UseCors("notesnook");
-            app.UseVersion(Servers.IdentityServer);
+            app.UseVersion();
 
             app.UseRouting();
 
@@ -203,9 +201,7 @@ namespace Streetwriters.Identity
 
             app.UseWamp(WampServers.IdentityServer, (realm, server) =>
             {
-                realm.Services.RegisterCallee(() => app.ApplicationServices.CreateScope().ServiceProvider.GetRequiredService<IUserAccountService>());
+                realm.Subscribe(server.Topics.CreateSubscriptionTopic, async (CreateSubscriptionMessage message) =>
-
-                realm.Subscribe(SubscriptionServerTopics.CreateSubscriptionTopic, async (CreateSubscriptionMessage message) =>
                 {
                     using (var serviceScope = app.ApplicationServices.CreateScope())
                     {
@@ -214,7 +210,7 @@ namespace Streetwriters.Identity
                         await MessageHandlers.CreateSubscription.Process(message, userManager);
                     }
                 });
-                realm.Subscribe(SubscriptionServerTopics.DeleteSubscriptionTopic, async (DeleteSubscriptionMessage message) =>
+                realm.Subscribe(server.Topics.DeleteSubscriptionTopic, async (DeleteSubscriptionMessage message) =>
                 {
                     using (var serviceScope = app.ApplicationServices.CreateScope())
                     {
@@ -240,7 +236,7 @@ namespace Streetwriters.Identity
                 cm.SetIgnoreExtraElements(true);
             });
 
-            services.AddSingleton<IPersistedGrantDbContext, CustomPersistedGrantDbContext>();
+            services.AddScoped<IPersistedGrantDbContext, CustomPersistedGrantDbContext>();
             services.AddTransient<IPersistedGrantStore, PersistedGrantStore>();
             services.AddTransient<TokenCleanup>();
 

Streetwriters.Identity/Streetwriters.Identity.csproj

@@ -1,8 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <StartupObject>Streetwriters.Identity.Program</StartupObject>
+    <LangVersion>10.0</LangVersion>
+    <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
+    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
   </PropertyGroup>
 
   <ItemGroup>
@@ -26,7 +33,6 @@
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.DataProtection" Version="7.0.4" />
     <PackageReference Include="AspNetCore.Identity.Mongo" Version="8.3.3" />
-    <PackageReference Include="Twilio" Version="6.13.0" />
     <PackageReference Include="WebMarkupMin.Core" Version="2.13.0" />
     <PackageReference Include="WebMarkupMin.NUglify" Version="2.12.0" />
   </ItemGroup>

Streetwriters.Identity/Validation/LockedOutValidationResult.cs

@@ -26,12 +26,11 @@ namespace Streetwriters.Identity.Validation
     {
         public LockedOutValidationResult(TimeSpan? timeLeft)
         {
-            Error = "locked_out";
+            base.Error = "locked_out";
-            IsError = true;
             if (timeLeft.HasValue)
-                ErrorDescription = $"You have been locked out. Please try again in {timeLeft?.Minutes.Pluralize("minute", "minutes")} and {timeLeft?.Seconds.Pluralize("second", "seconds")}.";
+                base.ErrorDescription = $"You have been locked out. Please try again in {timeLeft?.Minutes.Pluralize("minute", "minutes")} and {timeLeft?.Seconds.Pluralize("second", "seconds")}.";
             else
-                ErrorDescription = $"You have been locked out.";
+                base.ErrorDescription = $"You have been locked out.";
         }
     }
 }

Streetwriters.Identity/Validation/MFAGrantValidator.cs

@@ -89,6 +89,16 @@ namespace Streetwriters.Identity.Validation
             var user = await UserManager.FindByIdAsync(userId);
             if (user == null) return;
 
+            context.Result.Error = "invalid_mfa";
+            context.Result.ErrorDescription = "Please provide a valid multi-factor authentication code.";
+
+            if (string.IsNullOrEmpty(mfaCode)) return;
+            if (string.IsNullOrEmpty(mfaMethod))
+            {
+                context.Result.ErrorDescription = "Please provide a valid multi-factor authentication method.";
+                return;
+            }
+
             var isLockedOut = await UserManager.IsLockedOutAsync(user);
             if (isLockedOut)
             {
@@ -97,23 +107,19 @@ namespace Streetwriters.Identity.Validation
                 return;
             }
 
-            context.Result.Error = "invalid_mfa";
-            context.Result.ErrorDescription = "Please provide a valid multi-factor authentication code.";
-
-            if (!await UserManager.GetTwoFactorEnabledAsync(user))
-                await MFAService.EnableMFAAsync(user, MFAMethods.Email);
-
-            if (string.IsNullOrEmpty(mfaCode)) return;
-            if (string.IsNullOrEmpty(mfaMethod) || !MFAService.IsValidMFAMethod(mfaMethod))
-            {
-                context.Result.ErrorDescription = "Please provide a valid multi-factor authentication method.";
-                return;
-            }
-
             if (mfaMethod == MFAMethods.RecoveryCode)
             {
                 context.Result.ErrorDescription = "Please provide a valid multi-factor authentication recovery code.";
 
+                // This happens for new users who haven't set up 2FA yet; in which case
+                // we default to email. However, there are no recovery codes for that user
+                // yet.
+                // Without this, RedeemTwoFactorRecoveryCodeAsync succeeds with any recovery
+                // code (valid or invalid).
+                var isTwoFactorEnabled = await UserManager.GetTwoFactorEnabledAsync(user);
+                if (!isTwoFactorEnabled)
+                    return;
+
                 var result = await UserManager.RedeemTwoFactorRecoveryCodeAsync(user, mfaCode);
                 if (!result.Succeeded)
                 {
@@ -124,7 +130,9 @@ namespace Streetwriters.Identity.Validation
             }
             else
             {
-                if (!await MFAService.VerifyOTPAsync(user, mfaCode, mfaMethod))
+                var provider = mfaMethod == MFAMethods.Email || mfaMethod == MFAMethods.SMS ? TokenOptions.DefaultPhoneProvider : UserManager.Options.Tokens.AuthenticatorTokenProvider;
+                var isMFACodeValid = await MFAService.VerifyOTPAsync(user, mfaCode, mfaMethod);
+                if (!isMFACodeValid)
                 {
                     await UserManager.AccessFailedAsync(user);
                     await EmailSender.SendFailedLoginAlertAsync(user.Email, httpContext.GetClientInfo(), client).ConfigureAwait(false);
@@ -132,9 +140,8 @@ namespace Streetwriters.Identity.Validation
                 }
             }
 
-            await UserManager.ResetAccessFailedCountAsync(user);
             context.Result.IsError = false;
-            context.Result.Subject = await TokenGenerationService.TransformTokenRequestAsync(context.Request, user, GrantType, [Config.MFA_PASSWORD_GRANT_TYPE_SCOPE]);
+            context.Result.Subject = await TokenGenerationService.TransformTokenRequestAsync(context.Request, user, GrantType, new string[] { Config.MFA_PASSWORD_GRANT_TYPE_SCOPE });
         }
 
 

Streetwriters.Identity/Validation/MFAPasswordGrantValidator.cs

@@ -87,21 +87,18 @@ namespace Streetwriters.Identity.Validation
             if (user == null) return;
 
             var result = await SignInManager.CheckPasswordSignInAsync(user, password, true);
-
+            if (!result.Succeeded)
-            if (result.IsLockedOut)
+            {
+                await EmailSender.SendFailedLoginAlertAsync(user.Email, httpContext.GetClientInfo(), client).ConfigureAwait(false);
+                return;
+            }
+            else if (result.IsLockedOut)
             {
                 var timeLeft = user.LockoutEnd - DateTimeOffset.Now;
                 context.Result = new LockedOutValidationResult(timeLeft);
                 return;
             }
 
-            if (!result.Succeeded)
-            {
-                await EmailSender.SendFailedLoginAlertAsync(user.Email, httpContext.GetClientInfo(), client).ConfigureAwait(false);
-                return;
-            }
-
-            await UserManager.ResetAccessFailedCountAsync(user);
             var sub = await UserManager.GetUserIdAsync(user);
             context.Result = new GrantValidationResult(sub, AuthenticationMethods.Password);
         }

Streetwriters.Identity/appsettings.json

@@ -1,7 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Warning"
-    }
-  }
-}

Streetwriters.Messenger/Dockerfile

@@ -1,52 +1,28 @@
-FROM mcr.microsoft.com/dotnet/runtime-deps:8.0-alpine AS base
+FROM mcr.microsoft.com/dotnet/sdk:7.0 AS build
 WORKDIR /app
-EXPOSE 80
-EXPOSE 443
-
-FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-alpine AS build
-ARG TARGETARCH
-ARG BUILDPLATFORM
-ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
-
-WORKDIR /src
 
+# restore all project dependencies
 COPY Streetwriters.Data/*.csproj ./Streetwriters.Data/
+RUN dotnet restore /app/Streetwriters.Data/Streetwriters.Data.csproj --use-current-runtime
+
 COPY Streetwriters.Common/*.csproj ./Streetwriters.Common/
+RUN dotnet restore /app/Streetwriters.Common/Streetwriters.Common.csproj --use-current-runtime
+
 COPY Streetwriters.Messenger/*.csproj ./Streetwriters.Messenger/
+RUN dotnet restore /app/Streetwriters.Messenger/Streetwriters.Messenger.csproj --use-current-runtime
 
-# restore dependencies
+# copy everything else
-RUN dotnet restore -v d /src/Streetwriters.Messenger/Streetwriters.Messenger.csproj --use-current-runtime
-
 COPY Streetwriters.Data/ ./Streetwriters.Data/
 COPY Streetwriters.Common/ ./Streetwriters.Common/
 COPY Streetwriters.Messenger/ ./Streetwriters.Messenger/
 
-WORKDIR /src/Streetwriters.Messenger/
+# build
+WORKDIR /app/Streetwriters.Messenger/
+ENV DOTNET_TC_QuickJitForLoops="1" DOTNET_ReadyToRun="0" DOTNET_TieredPGO="1" DOTNET_SYSTEM_GLOBALIZATION_INVARIANT="true"
+RUN dotnet publish -c Release -o /app/out --use-current-runtime --self-contained false --no-restore
 
-RUN dotnet build -c Release -o /app/build -a $TARGETARCH
+# final stage/image
-
+FROM mcr.microsoft.com/dotnet/aspnet:7.0
-FROM build AS publish
-RUN dotnet publish -c Release -o /app/publish \
-  #--runtime alpine-x64 \
-  --self-contained true \
-  /p:TrimMode=partial \
-  /p:PublishTrimmed=true \
-  /p:PublishSingleFile=true \
-  /p:JsonSerializerIsReflectionEnabledByDefault=true \
-  -a $TARGETARCH
-
-FROM --platform=$BUILDPLATFORM base AS final
-ARG TARGETARCH
-ARG BUILDPLATFORM
-
-# create a new user and change directory ownership
-RUN adduser --disabled-password \
-  --home /app \
-  --gecos '' dotnetuser && chown -R dotnetuser /app
-
-# impersonate into the new user
-USER dotnetuser
 WORKDIR /app
-
+COPY --from=build /app/out .
-COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "Streetwriters.Messenger.dll"]
-ENTRYPOINT ["./Streetwriters.Messenger"]

Streetwriters.Messenger/Startup.cs

@@ -28,7 +28,6 @@ using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.ResponseCompression;
-using Microsoft.Extensions.Caching.Distributed;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -75,11 +74,11 @@ namespace Streetwriters.Messenger
                 options.Authority = Servers.IdentityServer.ToString();
                 options.ClientSecret = Constants.NOTESNOOK_API_SECRET;
                 options.ClientId = "notesnook";
-                options.DiscoveryPolicy.RequireHttps = false;
-                options.CacheKeyGenerator = (options, token) => (token + ":" + "reference_token").Sha256();
                 options.SaveToken = true;
                 options.EnableCaching = true;
                 options.CacheDuration = TimeSpan.FromMinutes(30);
+                // TODO
+                options.DiscoveryPolicy.RequireHttps = false;
             });
 
             services.AddServerSentEvents();
@@ -103,7 +102,7 @@ namespace Streetwriters.Messenger
             }
 
             app.UseCors("notesnook");
-            app.UseVersion(Servers.MessengerServer);
+            app.UseVersion();
 
             app.UseRouting();
 
@@ -120,7 +119,7 @@ namespace Streetwriters.Messenger
             app.UseWamp(WampServers.MessengerServer, (realm, server) =>
             {
                 IServerSentEventsService service = app.ApplicationServices.GetRequiredService<IServerSentEventsService>();
-                realm.Subscribe<SendSSEMessage>(MessengerServerTopics.SendSSETopic, async (ev) =>
+                realm.Subscribe<SendSSEMessage>(server.Topics.SendSSETopic, async (ev) =>
                 {
                     var message = JsonSerializer.Serialize(ev.Message);
                     if (ev.SendToAll)
@@ -132,9 +131,6 @@ namespace Streetwriters.Messenger
                         await SSEHelper.SendEventToUserAsync(message, service, ev.UserId, ev.OriginTokenId);
                     }
                 });
-
-                IDistributedCache cache = app.GetScopedService<IDistributedCache>();
-                realm.Subscribe<ClearCacheMessage>(IdentityServerTopics.ClearCacheTopic, (ev) => ev.Keys.ForEach((key) => cache.Remove(key)));
             });
 
             app.UseEndpoints(endpoints =>

Streetwriters.Messenger/Streetwriters.Messenger.csproj

@@ -1,22 +1,23 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <StartupObject>Streetwriters.Messenger.Program</StartupObject>
+    <LangVersion>10.0</LangVersion>
+    <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
+    <TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="DotNetEnv" Version="2.3.0" />
     <PackageReference Include="Lib.AspNetCore.ServerSentEvents" Version="6.0.0" />
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="5.0.0"
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="5.0.0" NoWarn="NU1605" />
-      NoWarn="NU1605" />
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="5.0.0" NoWarn="NU1605" />
-    <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="5.0.0"
+    <PackageReference Include="IdentityServer4.AccessTokenValidation" Version="3.0.1" />
-      NoWarn="NU1605" />
-    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="6.2.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Streetwriters.Common\Streetwriters.Common.csproj" />
   </ItemGroup>
 
-</Project>
+</Project>

Streetwriters.Messenger/appsettings.json

@@ -1,7 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Warning"
-    }
-  }
-}

docker-compose.yml

@@ -1,4 +1,7 @@
-x-server-discovery: &server-discovery
+version: "3.4"
+
+x-server-discovery:
+  &server-discovery
   NOTESNOOK_SERVER_PORT: 80
   NOTESNOOK_SERVER_HOST: notesnook-server
   IDENTITY_SERVER_PORT: 80
@@ -7,12 +10,13 @@ x-server-discovery: &server-discovery
   SSE_SERVER_HOST: sse-server
   SELF_HOSTED: 1
 
-x-env-files: &env-files
+x-env-files:
+  &env-files
   - .env
 
 services:
   notesnook-db:
-    image: mongo:7.0.12
+    image: mongo
     networks:
       - notesnook
     command: --replSet rs0 --bind_ip_all
@@ -23,7 +27,7 @@ services:
   # upgrading it to a replica set. This is only required once but we running
   # it multiple times is no issue.
   initiate-rs0:
-    image: mongo:7.0.12
+    image: mongo
     networks:
       - notesnook
     depends_on:
@@ -38,7 +42,7 @@ services:
         EOF
 
   notesnook-s3:
-    image: minio/minio:RELEASE.2024-07-29T22-14-52Z
+    image: minio/minio
     ports:
       - 9000:9000
       - 9090:9090
@@ -48,13 +52,14 @@ services:
       - ${HOME}/.notesnook/s3:/data/s3
     environment:
       MINIO_BROWSER: "on"
-    env_file: *env-files
+    env_file:
+      - ./.env.local
     command: server /data/s3 --console-address :9090
 
   # There's no way to specify a default bucket in Minio so we have to
   # set it up ourselves.
   setup-s3:
-    image: minio/mc:RELEASE.2024-07-26T13-08-44Z
+    image: minio/mc
     depends_on:
       - notesnook-s3
     networks:
@@ -64,13 +69,15 @@ services:
     command:
       - -c
       - |
-        until mc alias set minio http://notesnook-s3:9000 ${MINIO_ROOT_USER:-minioadmin} ${MINIO_ROOT_PASSWORD:-minioadmin}; do
+        until mc config host add minio http://notesnook-s3:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD; do
           sleep 1;
         done;
-        mc mb minio/$$S3_BUCKET_NAME -p
+        mc mb minio/nn-attachments -p
 
   identity-server:
-    image: streetwriters/identity:latest
+    build:
+      context: .
+      dockerfile: ./Streetwriters.Identity/Dockerfile
     ports:
       - "8264:80"
     networks:
@@ -78,19 +85,15 @@ services:
     env_file: *env-files
     depends_on:
       - notesnook-db
-    healthcheck:
-      test: curl --fail http://localhost:8264/health || exit 1
-      interval: 40s
-      timeout: 30s
-      retries: 3
-      start_period: 60s
     environment:
       <<: *server-discovery
       MONGODB_CONNECTION_STRING: mongodb://notesnook-db:27017/identity?replSet=rs0
       MONGODB_DATABASE_NAME: identity
 
   notesnook-server:
-    image: streetwriters/notesnook-sync:latest
+    build:
+      context: .
+      dockerfile: ./Notesnook.API/Dockerfile
     ports:
       - "5264:80"
     networks:
@@ -100,25 +103,20 @@ services:
       - notesnook-s3
       - setup-s3
       - identity-server
-    healthcheck:
-      test: curl --fail http://localhost:5264/health || exit 1
-      interval: 40s
-      timeout: 30s
-      retries: 3
-      start_period: 60s
     environment:
       <<: *server-discovery
-      MONGODB_CONNECTION_STRING: mongodb://notesnook-db:27017/?replSet=rs0
+      MONGODB_CONNECTION_STRING: mongodb://notesnook-db:27017/notesnook?replSet=rs0
       MONGODB_DATABASE_NAME: notesnook
-      S3_INTERNAL_SERVICE_URL: "${S3_SERVICE_URL:-http://notesnook-s3:9000}"
+      S3_INTERNAL_SERVICE_URL: http://notesnook-s3:9000
-      S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-${MINIO_ROOT_USER:-minioadmin}}"
+      S3_ACCESS_KEY_ID: "${MINIO_ROOT_USER:-minioadmin}"
-      S3_ACCESS_KEY: "${S3_ACCESS_KEY:-${MINIO_ROOT_PASSWORD:-minioadmin}}"
+      S3_ACCESS_KEY: "${MINIO_ROOT_PASSWORD:-minioadmin}"
-      S3_SERVICE_URL: "${S3_SERVICE_URL:-http://localhost:9000}"
+      S3_SERVICE_URL: http://localhost:9000
-      S3_REGION: "${S3_REGION:-us-east-1}"
+      S3_REGION: us-east-1
-      S3_BUCKET_NAME: "${S3_BUCKET_NAME}"
 
   sse-server:
-    image: streetwriters/sse:latest
+    build:
+      context: .
+      dockerfile: ./Streetwriters.Messenger/Dockerfile
     ports:
       - "7264:80"
     env_file: *env-files
@@ -127,24 +125,8 @@ services:
       - notesnook-server
     networks:
       - notesnook
-    healthcheck:
-      test: curl --fail http://localhost:7264/health || exit 1
-      interval: 40s
-      timeout: 30s
-      retries: 3
-      start_period: 60s
     environment:
       <<: *server-discovery
 
-  autoheal:
-    image: willfarrell/autoheal:latest
-    tty: true
-    restart: always
-    environment:
-      - AUTOHEAL_INTERVAL=60
-      - AUTOHEAL_START_PERIOD=300
-      - AUTOHEAL_DEFAULT_STOP_TIMEOUT=10
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
 networks:
   notesnook: