sync: use w1 write concern for device_ids_chunks collection

Notesnook.API/Controllers/MonographsController.cs

@@ -35,6 +35,8 @@ using Notesnook.API.Authorization;
 using Notesnook.API.Models;
 using Notesnook.API.Services;
 using Streetwriters.Common;
+using Streetwriters.Common.Accessors;
+using Streetwriters.Common.Enums;
 using Streetwriters.Common.Helpers;
 using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Messages;
@@ -46,7 +48,7 @@ namespace Notesnook.API.Controllers
     [ApiController]
     [Route("monographs")]
     [Authorize("Sync")]
-    public class MonographsController(Repository<Monograph> monographs, IURLAnalyzer analyzer, SyncDeviceService syncDeviceService, ILogger<MonographsController> logger) : ControllerBase
+    public class MonographsController(Repository<Monograph> monographs, IURLAnalyzer analyzer, SyncDeviceService syncDeviceService, WampServiceAccessor serviceAccessor, ILogger<MonographsController> logger) : ControllerBase
     {
         const string SVG_PIXEL = "<svg xmlns='http://www.w3.org/2000/svg' width='1' height='1'><circle r='9'/></svg>";
         private const int MAX_DOC_SIZE = 15 * 1024 * 1024;
@@ -107,7 +109,11 @@ namespace Notesnook.API.Controllers
                 if (existingMonograph != null && !existingMonograph.Deleted) return await UpdateAsync(deviceId, monograph);
 
                 if (monograph.EncryptedContent == null)
-                    monograph.CompressedContent = (await CleanupContentAsync(User, monograph.Content)).CompressBrotli();
+                {
+                    var sanitizationLevel = User.IsUserSubscribed() ? ContentSanitizationLevel.Partial : ContentSanitizationLevel.Full;
+                    monograph.CompressedContent = (await SanitizeContentAsync(monograph.Content, sanitizationLevel)).CompressBrotli();
+                    monograph.ContentSanitizationLevel = sanitizationLevel;
+                }
                 monograph.UserId = userId;
                 monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
 
@@ -158,8 +164,12 @@ namespace Notesnook.API.Controllers
                 if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE)
                     return base.BadRequest("Monograph is too big. Max allowed size is 15mb.");
 
+                var sanitizationLevel = ContentSanitizationLevel.Unknown;
                 if (monograph.EncryptedContent == null)
-                    monograph.CompressedContent = (await CleanupContentAsync(User, monograph.Content)).CompressBrotli();
+                {
+                    sanitizationLevel = User.IsUserSubscribed() ? ContentSanitizationLevel.Partial : ContentSanitizationLevel.Full;
+                    monograph.CompressedContent = (await SanitizeContentAsync(monograph.Content, sanitizationLevel)).CompressBrotli();
+                }
                 else
                     monograph.Content = null;
 
@@ -173,6 +183,7 @@ namespace Notesnook.API.Controllers
                     .Set(m => m.SelfDestruct, monograph.SelfDestruct)
                     .Set(m => m.Title, monograph.Title)
                     .Set(m => m.Password, monograph.Password)
+                    .Set(m => m.ContentSanitizationLevel, sanitizationLevel)
                 );
                 if (!result.IsAcknowledged) return BadRequest();
 
@@ -223,7 +234,22 @@ namespace Notesnook.API.Controllers
             }
 
             if (monograph.EncryptedContent == null)
+            {
+                var isContentUnsanitized = monograph.ContentSanitizationLevel == ContentSanitizationLevel.Partial || monograph.ContentSanitizationLevel == ContentSanitizationLevel.Unknown;
+                if (!Constants.IS_SELF_HOSTED && isContentUnsanitized && serviceAccessor.UserSubscriptionService != null && !await serviceAccessor.UserSubscriptionService.IsUserSubscribedAsync(Clients.Notesnook.Id, monograph.UserId!))
+                {
+                    var cleaned = await SanitizeContentAsync(monograph.CompressedContent?.DecompressBrotli(), ContentSanitizationLevel.Full);
+                    monograph.CompressedContent = cleaned.CompressBrotli();
+                    await monographs.Collection.UpdateOneAsync(
+                        CreateMonographFilter(monograph.UserId!, monograph),
+                        Builders<Monograph>.Update
+                            .Set(m => m.CompressedContent, monograph.CompressedContent)
+                            .Set(m => m.ContentSanitizationLevel, ContentSanitizationLevel.Full)
+                    );
+                }
                 monograph.Content = monograph.CompressedContent?.DecompressBrotli();
+            }
+
             monograph.ItemId ??= monograph.Id;
             return Ok(monograph);
         }
@@ -241,7 +267,7 @@ namespace Notesnook.API.Controllers
             if (monograph.SelfDestruct)
             {
                 await monographs.Collection.ReplaceOneAsync(
-                    CreateMonographFilter(monograph.UserId, monograph),
+                    CreateMonographFilter(monograph.UserId!, monograph),
                     new Monograph
                     {
                         ItemId = id,
@@ -251,12 +277,12 @@ namespace Notesnook.API.Controllers
                         ViewCount = 0
                     }
                 );
-                await MarkMonographForSyncAsync(monograph.UserId, id);
+                await MarkMonographForSyncAsync(monograph.UserId!, id);
             }
             else if (!hasVisitedBefore)
             {
                 await monographs.Collection.UpdateOneAsync(
-                    CreateMonographFilter(monograph.UserId, monograph),
+                    CreateMonographFilter(monograph.UserId!, monograph),
                     Builders<Monograph>.Update.Inc(m => m.ViewCount, 1)
                 );
 
@@ -329,7 +355,20 @@ namespace Notesnook.API.Controllers
             await syncDeviceService.AddIdsToAllDevicesAsync(userId, [new(monographId, "monograph")]);
         }
 
-        private async Task<string> CleanupContentAsync(ClaimsPrincipal user, string? content)
+        // (selector, url-bearing attribute) pairs to inspect
+        private static readonly (string Selector, string Attribute)[] urlElements =
+        [
+            ("a", "href"),
+            ("img", "src"),
+            ("iframe", "src"),
+            ("embed", "src"),
+            ("object", "data"),
+            ("source", "src"),
+            ("video", "src"),
+            ("audio", "src"),
+        ];
+
+        private async Task<string> SanitizeContentAsync(string? content, ContentSanitizationLevel level)
         {
             if (string.IsNullOrEmpty(content)) return string.Empty;
             if (Constants.IS_SELF_HOSTED) return content;
@@ -338,31 +377,36 @@ namespace Notesnook.API.Controllers
                 var json = JsonSerializer.Deserialize<MonographContent>(content) ?? throw new Exception("Invalid monograph content.");
                 var html = json.Data;
 
-                if (user.IsUserSubscribed())
+                if (level == ContentSanitizationLevel.Partial)
                 {
                     var config = Configuration.Default.WithDefaultLoader();
                     var context = BrowsingContext.New(config);
                     var document = await context.OpenAsync(r => r.Content(html));
-                    foreach (var element in document.QuerySelectorAll("a"))
+
+                    foreach (var (selector, attribute) in urlElements)
                     {
-                        var href = element.GetAttribute("href");
+                        foreach (var element in document.QuerySelectorAll(selector))
-                        if (string.IsNullOrEmpty(href)) continue;
-                        if (!await analyzer.IsURLSafeAsync(href))
                         {
-                            logger.LogInformation("Malicious URL detected: {Url}", href);
+                            var url = element.GetAttribute(attribute);
-                            element.RemoveAttribute("href");
+                            if (string.IsNullOrEmpty(url)) continue;
+                            if (!await analyzer.IsURLSafeAsync(url))
+                            {
+                                logger.LogInformation("Malicious URL detected in <{Selector} {Attribute}>: {Url}", selector, attribute, url);
+                                element.RemoveAttribute(attribute);
+                            }
                         }
                     }
+
                     html = document.ToHtml();
                 }
-                else
+                else if (level == ContentSanitizationLevel.Full)
                 {
                     var config = Configuration.Default.WithDefaultLoader();
                     var context = BrowsingContext.New(config);
                     var document = await context.OpenAsync(r => r.Content(html));
                     foreach (var element in document.QuerySelectorAll("a,iframe,img,object,svg,button,link"))
                     {
-                        foreach (var attr in element.Attributes)
+                        foreach (var attr in element.Attributes.ToList())
                             element.RemoveAttribute(attr.Name);
                     }
                     html = document.ToHtml();

Notesnook.API/Controllers/S3Controller.cs

@@ -210,25 +210,25 @@ namespace Notesnook.API.Controllers
             }
         }
 
-        [HttpPost("bulk-delete")]
+        // [HttpPost("bulk-delete")]
-        public async Task<IActionResult> DeleteBulkAsync([FromBody] DeleteBulkObjectsRequest request)
+        // public async Task<IActionResult> DeleteBulkAsync([FromBody] DeleteBulkObjectsRequest request)
-        {
+        // {
-            try
+        //     try
-            {
+        //     {
-                if (request.Names == null || request.Names.Length == 0)
+        //         if (request.Names == null || request.Names.Length == 0)
-                {
+        //         {
-                    return BadRequest(new { error = "No files specified for deletion." });
+        //             return BadRequest(new { error = "No files specified for deletion." });
-                }
+        //         }
 
-                var userId = this.User.GetUserId();
+        //         var userId = this.User.GetUserId();
-                await s3Service.DeleteObjectsAsync(userId, request.Names);
+        //         await s3Service.DeleteObjectsAsync(userId, request.Names);
-                return Ok();
+        //         return Ok();
-            }
+        //     }
-            catch (Exception ex)
+        //     catch (Exception ex)
-            {
+        //     {
-                logger.LogError(ex, "Error deleting objects for user.");
+        //         logger.LogError(ex, "Error deleting objects for user.");
-                return BadRequest(new { error = "Failed to delete attachments." });
+        //         return BadRequest(new { error = "Failed to delete attachments." });
-            }
+        //     }
-        }
+        // }
     }
 }

Notesnook.API/Models/ContentSanitizationLevel.cs

@@ -0,0 +1,38 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Notesnook.API.Models
+{
+    public enum ContentSanitizationLevel
+    {
+        Unknown = 0,
+        /// <summary>
+        /// Full sanitization applied: links, iframes, images, and other embeds are stripped.
+        /// Applied to monographs published by free-tier users.
+        /// </summary>
+        Full = 1,
+
+        /// <summary>
+        /// Partial sanitization: only unsafe/malicious URLs are removed; rich content is preserved.
+        /// Applied to monographs published by subscribed users. Requires re-sanitization if the
+        /// publisher's subscription lapses.
+        /// </summary>
+        Partial = 2
+    }
+}

Notesnook.API/Models/Monograph.cs

@@ -83,5 +83,8 @@ namespace Notesnook.API.Models
 
         [JsonPropertyName("viewCount")]
         public int ViewCount { get; set; }
+
+        [JsonIgnore]
+        public ContentSanitizationLevel ContentSanitizationLevel { get; set; }
     }
 }

Notesnook.API/Notesnook.API.csproj

@@ -17,6 +17,7 @@
     <PackageReference Include="AspNetCore.HealthChecks.MongoDb" Version="6.0.1-rc2.2" />
     <PackageReference Include="AWSSDK.S3" Version="3.7.310.8" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="6.0.3" />
+    <PackageReference Include="Microsoft.AspNetCore.SignalR.StackExchangeRedis" Version="9.0.13" />
     <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Https" Version="2.2.0" />
     <PackageReference Include="Nanoid" Version="3.1.0" />
     <PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-alpha.2" />

Notesnook.API/Services/SyncDeviceService.cs

@@ -82,7 +82,7 @@ namespace Notesnook.API.Services
             if (chunk != null)
             {
                 var update = Builders<DeviceIdsChunk>.Update.AddToSetEach(x => x.Ids, ids.Select(i => i.ToString()));
-                await repositories.DeviceIdsChunks.Collection.UpdateOneAsync(
+                await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).UpdateOneAsync(
                     Builders<DeviceIdsChunk>.Filter.Eq(x => x.Id, chunk.Id),
                     update
                 );
@@ -96,11 +96,11 @@ namespace Notesnook.API.Services
                     Key = key,
                     Ids = [.. ids.Select(i => i.ToString())]
                 };
-                await repositories.DeviceIdsChunks.Collection.InsertOneAsync(newChunk);
+                await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).InsertOneAsync(newChunk);
             }
 
             var emptyChunksFilter = DeviceIdsChunkFilter(userId, deviceId, key) & Builders<DeviceIdsChunk>.Filter.Size(x => x.Ids, 0);
-            await repositories.DeviceIdsChunks.Collection.DeleteManyAsync(emptyChunksFilter);
+            await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).DeleteManyAsync(emptyChunksFilter);
         }
 
         public async Task WriteIdsAsync(string userId, string deviceId, string key, IEnumerable<ItemKey> ids)
@@ -121,7 +121,7 @@ namespace Notesnook.API.Services
                 };
                 writes.Add(new InsertOneModel<DeviceIdsChunk>(newChunk));
             }
-            await repositories.DeviceIdsChunks.Collection.BulkWriteAsync(writes);
+            await repositories.DeviceIdsChunks.Collection.WithWriteConcern(WriteConcern.W1).BulkWriteAsync(writes);
         }
 
         public async Task<HashSet<ItemKey>> FetchUnsyncedIdsAsync(string userId, string deviceId)

Notesnook.API/Startup.cs

@@ -210,13 +210,16 @@ namespace Notesnook.API
 
             services.AddHealthChecks();
 
-            services.AddSignalR((hub) =>
+            var signalR = services.AddSignalR((hub) =>
             {
                 hub.MaximumReceiveMessageSize = 100 * 1024 * 1024;
                 hub.ClientTimeoutInterval = TimeSpan.FromMinutes(10);
                 hub.EnableDetailedErrors = true;
             }).AddMessagePackProtocol().AddJsonProtocol();
 
+            if (!string.IsNullOrEmpty(Constants.SIGNALR_REDIS_CONNECTION_STRING))
+                signalR.AddStackExchangeRedis(Constants.SIGNALR_REDIS_CONNECTION_STRING);
+
             services.AddResponseCompression(options =>
             {
                 options.EnableForHttps = true;

Streetwriters.Common/Constants.cs

@@ -79,6 +79,7 @@ namespace Streetwriters.Common
         public static string? SUBSCRIPTIONS_CERT_PATH => ReadSecret("SUBSCRIPTIONS_CERT_PATH");
         public static string? SUBSCRIPTIONS_CERT_KEY_PATH => ReadSecret("SUBSCRIPTIONS_CERT_KEY_PATH");
         public static string[] NOTESNOOK_CORS_ORIGINS => ReadSecret("NOTESNOOK_CORS")?.Split(",") ?? [];
+        public static string? SIGNALR_REDIS_CONNECTION_STRING => ReadSecret("SIGNALR_REDIS_CONNECTION_STRING");
 
         public static string? ReadSecret(string name)
         {

Streetwriters.Common/Interfaces/IUserSubscriptionService.cs

@@ -9,6 +9,8 @@ namespace Streetwriters.Common.Interfaces
     {
         [WampProcedure("co.streetwriters.subscriptions.subscriptions.get_user_subscription")]
         Task<Subscription?> GetUserSubscriptionAsync(string clientId, string userId);
+        [WampProcedure("co.streetwriters.subscriptions.subscriptions.is_user_subscribed")]
+        Task<bool> IsUserSubscribedAsync(string clientId, string userId);
         Subscription TransformUserSubscription(Subscription subscription);
     }
 }

Streetwriters.Common/Services/URLAnalyzer.cs

@@ -1,6 +1,9 @@
 using System;
 using System.Net.Http;
+using System.Net.Http.Headers;
 using System.Net.Http.Json;
+using System.Text;
+using System.Text.Json;
 using System.Threading.Tasks;
 using Streetwriters.Common.Interfaces;
 
@@ -22,7 +25,8 @@ namespace Streetwriters.Common.Services
         public async Task<bool> IsURLSafeAsync(string uri)
         {
             if (string.IsNullOrEmpty(Constants.WEBRISK_API_URI)) return true;
-            var response = await httpClient.PostAsJsonAsync(Constants.WEBRISK_API_URI, new { uri });
+            var body = new StringContent(JsonSerializer.Serialize(new { uri }), Encoding.UTF8, new MediaTypeHeaderValue("application/json"));
+            var response = await httpClient.PostAsync(Constants.WEBRISK_API_URI, body);
             if (!response.IsSuccessStatusCode) return true;
             var json = await response.Content.ReadFromJsonAsync<WebRiskAPIResponse>();
             return json.Threat.ThreatTypes == null || json.Threat.ThreatTypes.Length == 0;