diff --git a/Notesnook.API/Controllers/MonographsController.cs b/Notesnook.API/Controllers/MonographsController.cs index 7e1ff9c..5a318eb 100644 --- a/Notesnook.API/Controllers/MonographsController.cs +++ b/Notesnook.API/Controllers/MonographsController.cs @@ -35,6 +35,8 @@ using Notesnook.API.Authorization; using Notesnook.API.Models; using Notesnook.API.Services; using Streetwriters.Common; +using Streetwriters.Common.Accessors; +using Streetwriters.Common.Enums; using Streetwriters.Common.Helpers; using Streetwriters.Common.Interfaces; using Streetwriters.Common.Messages; @@ -46,7 +48,7 @@ namespace Notesnook.API.Controllers [ApiController] [Route("monographs")] [Authorize("Sync")] - public class MonographsController(Repository monographs, IURLAnalyzer analyzer, SyncDeviceService syncDeviceService, ILogger logger) : ControllerBase + public class MonographsController(Repository monographs, IURLAnalyzer analyzer, SyncDeviceService syncDeviceService, WampServiceAccessor serviceAccessor, ILogger logger) : ControllerBase { const string SVG_PIXEL = ""; private const int MAX_DOC_SIZE = 15 * 1024 * 1024; @@ -107,7 +109,11 @@ namespace Notesnook.API.Controllers if (existingMonograph != null && !existingMonograph.Deleted) return await UpdateAsync(deviceId, monograph); if (monograph.EncryptedContent == null) - monograph.CompressedContent = (await CleanupContentAsync(User, monograph.Content)).CompressBrotli(); + { + var sanitizationLevel = User.IsUserSubscribed() ? ContentSanitizationLevel.Partial : ContentSanitizationLevel.Full; + monograph.CompressedContent = (await SanitizeContentAsync(monograph.Content, sanitizationLevel)).CompressBrotli(); + monograph.ContentSanitizationLevel = sanitizationLevel; + } monograph.UserId = userId; monograph.DatePublished = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); @@ -158,8 +164,12 @@ namespace Notesnook.API.Controllers if (monograph.EncryptedContent?.Cipher.Length > MAX_DOC_SIZE || monograph.CompressedContent?.Length > MAX_DOC_SIZE) return base.BadRequest("Monograph is too big. Max allowed size is 15mb."); + var sanitizationLevel = ContentSanitizationLevel.Unknown; if (monograph.EncryptedContent == null) - monograph.CompressedContent = (await CleanupContentAsync(User, monograph.Content)).CompressBrotli(); + { + sanitizationLevel = User.IsUserSubscribed() ? ContentSanitizationLevel.Partial : ContentSanitizationLevel.Full; + monograph.CompressedContent = (await SanitizeContentAsync(monograph.Content, sanitizationLevel)).CompressBrotli(); + } else monograph.Content = null; @@ -173,6 +183,7 @@ namespace Notesnook.API.Controllers .Set(m => m.SelfDestruct, monograph.SelfDestruct) .Set(m => m.Title, monograph.Title) .Set(m => m.Password, monograph.Password) + .Set(m => m.ContentSanitizationLevel, sanitizationLevel) ); if (!result.IsAcknowledged) return BadRequest(); @@ -223,7 +234,22 @@ namespace Notesnook.API.Controllers } if (monograph.EncryptedContent == null) + { + var isContentUnsanitized = monograph.ContentSanitizationLevel == ContentSanitizationLevel.Partial || monograph.ContentSanitizationLevel == ContentSanitizationLevel.Unknown; + if (!Constants.IS_SELF_HOSTED && isContentUnsanitized && serviceAccessor.UserSubscriptionService != null && !await serviceAccessor.UserSubscriptionService.IsUserSubscribedAsync(Clients.Notesnook.Id, monograph.UserId!)) + { + var cleaned = await SanitizeContentAsync(monograph.CompressedContent?.DecompressBrotli(), ContentSanitizationLevel.Full); + monograph.CompressedContent = cleaned.CompressBrotli(); + await monographs.Collection.UpdateOneAsync( + CreateMonographFilter(monograph.UserId!, monograph), + Builders.Update + .Set(m => m.CompressedContent, monograph.CompressedContent) + .Set(m => m.ContentSanitizationLevel, ContentSanitizationLevel.Full) + ); + } monograph.Content = monograph.CompressedContent?.DecompressBrotli(); + } + monograph.ItemId ??= monograph.Id; return Ok(monograph); } @@ -241,7 +267,7 @@ namespace Notesnook.API.Controllers if (monograph.SelfDestruct) { await monographs.Collection.ReplaceOneAsync( - CreateMonographFilter(monograph.UserId, monograph), + CreateMonographFilter(monograph.UserId!, monograph), new Monograph { ItemId = id, @@ -251,12 +277,12 @@ namespace Notesnook.API.Controllers ViewCount = 0 } ); - await MarkMonographForSyncAsync(monograph.UserId, id); + await MarkMonographForSyncAsync(monograph.UserId!, id); } else if (!hasVisitedBefore) { await monographs.Collection.UpdateOneAsync( - CreateMonographFilter(monograph.UserId, monograph), + CreateMonographFilter(monograph.UserId!, monograph), Builders.Update.Inc(m => m.ViewCount, 1) ); @@ -329,7 +355,20 @@ namespace Notesnook.API.Controllers await syncDeviceService.AddIdsToAllDevicesAsync(userId, [new(monographId, "monograph")]); } - private async Task CleanupContentAsync(ClaimsPrincipal user, string? content) + // (selector, url-bearing attribute) pairs to inspect + private static readonly (string Selector, string Attribute)[] urlElements = + [ + ("a", "href"), + ("img", "src"), + ("iframe", "src"), + ("embed", "src"), + ("object", "data"), + ("source", "src"), + ("video", "src"), + ("audio", "src"), + ]; + + private async Task SanitizeContentAsync(string? content, ContentSanitizationLevel level) { if (string.IsNullOrEmpty(content)) return string.Empty; if (Constants.IS_SELF_HOSTED) return content; @@ -338,31 +377,36 @@ namespace Notesnook.API.Controllers var json = JsonSerializer.Deserialize(content) ?? throw new Exception("Invalid monograph content."); var html = json.Data; - if (user.IsUserSubscribed()) + if (level == ContentSanitizationLevel.Full) { var config = Configuration.Default.WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(r => r.Content(html)); - foreach (var element in document.QuerySelectorAll("a")) + + foreach (var (selector, attribute) in urlElements) { - var href = element.GetAttribute("href"); - if (string.IsNullOrEmpty(href)) continue; - if (!await analyzer.IsURLSafeAsync(href)) + foreach (var element in document.QuerySelectorAll(selector)) { - logger.LogInformation("Malicious URL detected: {Url}", href); - element.RemoveAttribute("href"); + var url = element.GetAttribute(attribute); + if (string.IsNullOrEmpty(url)) continue; + if (!await analyzer.IsURLSafeAsync(url)) + { + logger.LogInformation("Malicious URL detected in <{Selector} {Attribute}>: {Url}", selector, attribute, url); + element.RemoveAttribute(attribute); + } } } + html = document.ToHtml(); } - else + else if (level == ContentSanitizationLevel.Full) { var config = Configuration.Default.WithDefaultLoader(); var context = BrowsingContext.New(config); var document = await context.OpenAsync(r => r.Content(html)); foreach (var element in document.QuerySelectorAll("a,iframe,img,object,svg,button,link")) { - foreach (var attr in element.Attributes) + foreach (var attr in element.Attributes.ToList()) element.RemoveAttribute(attr.Name); } html = document.ToHtml(); diff --git a/Notesnook.API/Models/ContentSanitizationLevel.cs b/Notesnook.API/Models/ContentSanitizationLevel.cs new file mode 100644 index 0000000..4ff92fc --- /dev/null +++ b/Notesnook.API/Models/ContentSanitizationLevel.cs @@ -0,0 +1,38 @@ +/* +This file is part of the Notesnook Sync Server project (https://notesnook.com/) + +Copyright (C) 2023 Streetwriters (Private) Limited + +This program is free software: you can redistribute it and/or modify +it under the terms of the Affero GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +Affero GNU General Public License for more details. + +You should have received a copy of the Affero GNU General Public License +along with this program. If not, see . +*/ + +namespace Notesnook.API.Models +{ + public enum ContentSanitizationLevel + { + Unknown = 0, + /// + /// Full sanitization applied: links, iframes, images, and other embeds are stripped. + /// Applied to monographs published by free-tier users. + /// + Full = 1, + + /// + /// Partial sanitization: only unsafe/malicious URLs are removed; rich content is preserved. + /// Applied to monographs published by subscribed users. Requires re-sanitization if the + /// publisher's subscription lapses. + /// + Partial = 2 + } +} diff --git a/Notesnook.API/Models/Monograph.cs b/Notesnook.API/Models/Monograph.cs index 3d0aa59..147e8e0 100644 --- a/Notesnook.API/Models/Monograph.cs +++ b/Notesnook.API/Models/Monograph.cs @@ -83,5 +83,8 @@ namespace Notesnook.API.Models [JsonPropertyName("viewCount")] public int ViewCount { get; set; } + + [JsonIgnore] + public ContentSanitizationLevel ContentSanitizationLevel { get; set; } } } \ No newline at end of file diff --git a/Streetwriters.Common/Interfaces/IUserSubscriptionService.cs b/Streetwriters.Common/Interfaces/IUserSubscriptionService.cs index ee50ac2..a0d4a64 100644 --- a/Streetwriters.Common/Interfaces/IUserSubscriptionService.cs +++ b/Streetwriters.Common/Interfaces/IUserSubscriptionService.cs @@ -9,6 +9,8 @@ namespace Streetwriters.Common.Interfaces { [WampProcedure("co.streetwriters.subscriptions.subscriptions.get_user_subscription")] Task GetUserSubscriptionAsync(string clientId, string userId); + [WampProcedure("co.streetwriters.subscriptions.subscriptions.is_user_subscribed")] + Task IsUserSubscribedAsync(string clientId, string userId); Subscription TransformUserSubscription(Subscription subscription); } } \ No newline at end of file diff --git a/Streetwriters.Common/Services/URLAnalyzer.cs b/Streetwriters.Common/Services/URLAnalyzer.cs index 496d958..01583ea 100644 --- a/Streetwriters.Common/Services/URLAnalyzer.cs +++ b/Streetwriters.Common/Services/URLAnalyzer.cs @@ -1,6 +1,9 @@ using System; using System.Net.Http; +using System.Net.Http.Headers; using System.Net.Http.Json; +using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Streetwriters.Common.Interfaces; @@ -22,7 +25,8 @@ namespace Streetwriters.Common.Services public async Task IsURLSafeAsync(string uri) { if (string.IsNullOrEmpty(Constants.WEBRISK_API_URI)) return true; - var response = await httpClient.PostAsJsonAsync(Constants.WEBRISK_API_URI, new { uri }); + var body = new StringContent(JsonSerializer.Serialize(new { uri }), Encoding.UTF8, new MediaTypeHeaderValue("application/json")); + var response = await httpClient.PostAsync(Constants.WEBRISK_API_URI, body); if (!response.IsSuccessStatusCode) return true; var json = await response.Content.ReadFromJsonAsync(); return json.Threat.ThreatTypes == null || json.Threat.ThreatTypes.Length == 0;