s3: add bulk delete api

Notesnook.API/Controllers/S3Controller.cs

@@ -21,20 +21,17 @@ using System;
 using System.Net.Http;
 using System.Security.Claims;
 using System.Threading.Tasks;
-using Amazon.S3.Model;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http.Extensions;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
-using Notesnook.API.Accessors;
 using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Streetwriters.Common;
 using Streetwriters.Common.Accessors;
 using Streetwriters.Common.Extensions;
-using Streetwriters.Common.Interfaces;
 using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Controllers
@@ -212,5 +209,26 @@ namespace Notesnook.API.Controllers
                 return BadRequest(new { error = "Failed to delete attachment." });
             }
         }
+
+        [HttpPost("bulk-delete")]
+        public async Task<IActionResult> DeleteBulkAsync([FromBody] DeleteBulkObjectsRequest request)
+        {
+            try
+            {
+                if (request.Names == null || request.Names.Length == 0)
+                {
+                    return BadRequest(new { error = "No files specified for deletion." });
+                }
+
+                var userId = this.User.GetUserId();
+                await s3Service.DeleteObjectsAsync(userId, request.Names);
+                return Ok();
+            }
+            catch (Exception ex)
+            {
+                logger.LogError(ex, "Error deleting objects for user.");
+                return BadRequest(new { error = "Failed to delete attachments." });
+            }
+        }
     }
 }

Notesnook.API/Interfaces/IS3Service.cs

@@ -17,18 +17,16 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-using System.Threading;
 using System.Threading.Tasks;
 using Amazon.S3.Model;
 using Notesnook.API.Models;
-using Notesnook.API.Models.Responses;
-using Streetwriters.Common.Interfaces;
 
 namespace Notesnook.API.Interfaces
 {
     public interface IS3Service
     {
         Task DeleteObjectAsync(string userId, string name);
+        Task DeleteObjectsAsync(string userId, string[] names);
         Task DeleteDirectoryAsync(string userId);
         Task<long> GetObjectSizeAsync(string userId, string name);
         Task<string?> GetUploadObjectUrlAsync(string userId, string name);

Notesnook.API/Models/DeleteBulkRequest.cs

@@ -0,0 +1,25 @@
+/*
+This file is part of the Notesnook Sync Server project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the Affero GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+Affero GNU General Public License for more details.
+
+You should have received a copy of the Affero GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+namespace Notesnook.API.Models;
+
+public class DeleteBulkObjectsRequest
+{
+    public required string[] Names { get; set; }
+}

Notesnook.API/Services/S3Service.cs

@@ -24,21 +24,15 @@ using System.Net.Http;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Amazon;
-using Amazon.Runtime;
 using Amazon.S3;
 using Amazon.S3.Model;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
 using MongoDB.Driver;
-using Notesnook.API.Accessors;
 using Notesnook.API.Helpers;
 using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
 using Streetwriters.Common;
 using Streetwriters.Common.Accessors;
-using Streetwriters.Common.Enums;
-using Streetwriters.Common.Interfaces;
-using Streetwriters.Common.Models;
 
 namespace Notesnook.API.Services
 {
@@ -110,6 +104,70 @@ namespace Notesnook.API.Services
                 throw new Exception("Could not delete object.");
         }
 
+        public async Task DeleteObjectsAsync(string userId, string[] names)
+        {
+            var objectsToDelete = new List<KeyVersion>();
+
+            foreach (var name in names)
+            {
+                var objectName = GetFullObjectName(userId, name);
+                if (objectName == null) continue;
+
+                objectsToDelete.Add(new KeyVersion { Key = objectName });
+            }
+
+            if (objectsToDelete.Count == 0)
+            {
+                return;
+            }
+
+            // S3 DeleteObjectsRequest supports max 1000 keys per request
+            var batchSize = 1000;
+            var deleteErrors = new List<DeleteError>();
+            var failedBatches = 0;
+
+            for (int i = 0; i < objectsToDelete.Count; i += batchSize)
+            {
+                var batch = objectsToDelete.Skip(i).Take(batchSize).ToList();
+                var deleteObjectsResponse = await S3InternalClient.ExecuteWithFailoverAsync(
+                    (client) => client.DeleteObjectsAsync(new DeleteObjectsRequest
+                    {
+                        BucketName = INTERNAL_BUCKET_NAME,
+                        Objects = batch,
+                    }),
+                    operationName: "DeleteObjects",
+                    isWriteOperation: true
+                );
+
+                if (!IsSuccessStatusCode((int)deleteObjectsResponse.HttpStatusCode))
+                {
+                    failedBatches++;
+                }
+
+                if (deleteObjectsResponse.DeleteErrors.Count > 0)
+                {
+                    deleteErrors.AddRange(deleteObjectsResponse.DeleteErrors);
+                }
+            }
+
+            if (failedBatches > 0 || deleteErrors.Count > 0)
+            {
+                var errorParts = new List<string>();
+
+                if (failedBatches > 0)
+                {
+                    errorParts.Add($"{failedBatches} batch(es) failed with unsuccessful status code");
+                }
+
+                if (deleteErrors.Count > 0)
+                {
+                    errorParts.Add(string.Join(", ", deleteErrors.Select(e => $"{e.Key}: {e.Message}")));
+                }
+
+                throw new Exception(string.Join("; ", errorParts));
+            }
+        }
+
         public async Task DeleteDirectoryAsync(string userId)
         {
             var request = new ListObjectsV2Request

Streetwriters.Data/DbContexts/MongoDbContext.cs

@@ -33,9 +33,6 @@ namespace Streetwriters.Data.DbContexts
         public static IMongoClient CreateMongoDbClient(IDbSettings dbSettings)
         {
             var settings = MongoClientSettings.FromConnectionString(dbSettings.ConnectionString);
-            settings.MaxConnectionPoolSize = 500;
-            settings.MinConnectionPoolSize = 0;
-            settings.HeartbeatInterval = TimeSpan.FromSeconds(60);
             return new MongoClient(settings);
         }
 

Streetwriters.Identity/Controllers/AccountController.cs

@@ -97,12 +97,12 @@ namespace Streetwriters.Identity.Controllers
                     }
                 case TokenType.RESET_PASSWORD:
                     {
-                        if (!await UserManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword", code))
-                            return BadRequest("Invalid token.");
+                        // if (!await UserManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword", code))
+                        return BadRequest("Password reset is temporarily disabled due to some issues. It should be back soon. We apologize for the inconvenience.");
 
-                        var authorizationCode = await UserManager.GenerateUserTokenAsync(user, TokenOptions.DefaultProvider, "PasswordResetAuthorizationCode");
-                        var redirectUrl = $"{client.AccountRecoveryRedirectURL}?userId={userId}&code={authorizationCode}";
-                        return RedirectPermanent(redirectUrl);
+                        // var authorizationCode = await UserManager.GenerateUserTokenAsync(user, TokenOptions.DefaultProvider, "PasswordResetAuthorizationCode");
+                        // var redirectUrl = $"{client.AccountRecoveryRedirectURL}?userId={userId}&code={authorizationCode}";
+                        // return RedirectPermanent(redirectUrl);
                     }
                 default:
                     return BadRequest("Invalid type.");
@@ -149,21 +149,22 @@ namespace Streetwriters.Identity.Controllers
         [EnableRateLimiting("strict")]
         public async Task<IActionResult> ResetUserPassword([FromForm] ResetPasswordForm form)
         {
-            var client = Clients.FindClientById(form.ClientId);
-            if (client == null) return BadRequest("Invalid client_id.");
+            return BadRequest(new { error = "Password reset is temporarily disabled due to some issues. It should be back soon. We apologize for the inconvenience." });
+            //             var client = Clients.FindClientById(form.ClientId);
+            //             if (client == null) return BadRequest("Invalid client_id.");
 
-            var user = await UserManager.FindByEmailAsync(form.Email) ?? throw new Exception("User not found.");
-            if (!await UserService.IsUserValidAsync(UserManager, user, form.ClientId)) return Ok();
+            //             var user = await UserManager.FindByEmailAsync(form.Email) ?? throw new Exception("User not found.");
+            //             if (!await UserService.IsUserValidAsync(UserManager, user, form.ClientId)) return Ok();
 
-            var code = await UserManager.GenerateUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword");
-            var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.RESET_PASSWORD);
-#if (DEBUG || STAGING)
-            return Ok(callbackUrl);
-#else
-            logger.LogInformation("Password reset email sent to: {Email}, callback URL: {CallbackUrl}", user.Email, callbackUrl);
-            await EmailSender.SendPasswordResetEmailAsync(user.Email, callbackUrl, client);
-            return Ok();
-#endif
+            //             var code = await UserManager.GenerateUserTokenAsync(user, TokenOptions.DefaultProvider, "ResetPassword");
+            //             var callbackUrl = Url.TokenLink(user.Id.ToString(), code, client.Id, TokenType.RESET_PASSWORD);
+            // #if (DEBUG || STAGING)
+            //             return Ok(callbackUrl);
+            // #else
+            //             logger.LogInformation("Password reset email sent to: {Email}, callback URL: {CallbackUrl}", user.Email, callbackUrl);
+            //             await EmailSender.SendPasswordResetEmailAsync(user.Email, callbackUrl, client);
+            //             return Ok();
+            // #endif
         }
 
         [HttpPost("logout")]
@@ -250,31 +251,33 @@ namespace Streetwriters.Identity.Controllers
                     }
                 case "change_password":
                     {
-                        ArgumentNullException.ThrowIfNull(form.OldPassword);
-                        ArgumentNullException.ThrowIfNull(form.NewPassword);
-                        var result = await UserManager.ChangePasswordAsync(user, form.OldPassword, form.NewPassword);
-                        if (result.Succeeded)
-                        {
-                            await SendLogoutMessageAsync(user.Id.ToString(), "Password changed.");
-                            return Ok();
-                        }
-                        return BadRequest(result.Errors.ToErrors());
+                        return BadRequest(new { error = "Password change is temporarily disabled due to some issues. It should be back soon. We apologize for the inconvenience." });
+                        // ArgumentNullException.ThrowIfNull(form.OldPassword);
+                        // ArgumentNullException.ThrowIfNull(form.NewPassword);
+                        // var result = await UserManager.ChangePasswordAsync(user, form.OldPassword, form.NewPassword);
+                        // if (result.Succeeded)
+                        // {
+                        //     await SendLogoutMessageAsync(user.Id.ToString(), "Password changed.");
+                        //     return Ok();
+                        // }
+                        // return BadRequest(result.Errors.ToErrors());
                     }
                 case "reset_password":
                     {
-                        ArgumentNullException.ThrowIfNull(form.NewPassword);
-                        var result = await UserManager.RemovePasswordAsync(user);
-                        if (result.Succeeded)
-                        {
-                            await MFAService.ResetMFAAsync(user);
-                            result = await UserManager.AddPasswordAsync(user, form.NewPassword);
-                            if (result.Succeeded)
-                            {
-                                await SendLogoutMessageAsync(user.Id.ToString(), "Password reset.");
-                                return Ok();
-                            }
-                        }
-                        return BadRequest(result.Errors.ToErrors());
+                        return BadRequest(new { error = "Password reset is temporarily disabled due to some issues. It should be back soon. We apologize for the inconvenience." });
+                        // ArgumentNullException.ThrowIfNull(form.NewPassword);
+                        // var result = await UserManager.RemovePasswordAsync(user);
+                        // if (result.Succeeded)
+                        // {
+                        //     await MFAService.ResetMFAAsync(user);
+                        //     result = await UserManager.AddPasswordAsync(user, form.NewPassword);
+                        //     if (result.Succeeded)
+                        //     {
+                        //         await SendLogoutMessageAsync(user.Id.ToString(), "Password reset.");
+                        //         return Ok();
+                        //     }
+                        // }
+                        // return BadRequest(result.Errors.ToErrors());
                     }
                 case "change_marketing_consent":
                     {

Streetwriters.Identity/Startup.cs

@@ -53,6 +53,7 @@ using Streetwriters.Identity.Interfaces;
 using Streetwriters.Identity.Jobs;
 using Streetwriters.Identity.Services;
 using Streetwriters.Identity.Validation;
+using IdentityServer4.MongoDB.Configuration;
 
 namespace Streetwriters.Identity
 {
@@ -107,11 +108,6 @@ namespace Streetwriters.Identity
                 options.UsersCollection = "users";
                 // options.MigrationCollection = "migration";
                 options.ConnectionString = connectionString;
-                options.ClusterConfigurator = builder =>
-                {
-                    builder.ConfigureConnectionPool((c) => c.With(maxConnections: 500, minConnections: 0));
-                    builder.ConfigureServer(s => s.With(heartbeatInterval: TimeSpan.FromSeconds(60)));
-                };
             }).AddDefaultTokenProviders();
 
             services.AddIdentityServer(
@@ -137,6 +133,11 @@ namespace Streetwriters.Identity
             .AddKeyManagement()
             .AddFileSystemPersistence(Path.Combine(WebHostEnvironment.ContentRootPath, @"keystore"));
 
+            services.Configure<MongoDBConfiguration>(options =>
+            {
+                options.ConnectionString = connectionString;
+            });
+
             services.Configure<DataProtectionTokenProviderOptions>(options =>
             {
                 options.TokenLifespan = TimeSpan.FromHours(2);

cors-proxy/src/index.ts

@@ -210,7 +210,21 @@ const server = Bun.serve({
       });
     }
 
-    // Proxy the request
+    // Check if it's a YouTube URL and redirect instead of proxying
+    if (isYouTubeEmbed(targetUrl)) {
+      // YouTube URL detected, redirect to youtube-nocookie.com
+      logRequest(req.method, targetUrl, 200);
+      return new Response(serveYouTubeEmbed(targetUrl), {
+        status: 200,
+        headers: {
+          "Content-Type": "text/html; charset=utf-8",
+          "Content-Security-Policy": "frame-ancestors *",
+          "X-Frame-Options": "ALLOWALL",
+        },
+      });
+    }
+
+    // Proxy the request for non-YouTube URLs
     const response = await proxyRequest(targetUrl);
     logRequest(req.method, targetUrl, response.status);
     return response;
@@ -229,3 +243,80 @@ console.log(
 );
 console.log(`📋 Health check: http://${server.hostname}:${server.port}/health`);
 console.log(`🌍 Environment: ${Bun.env.NODE_ENV || "development"}`);
+
+/**
+ * This is required to bypass YouTube's Referrer Policy restrictions when
+ * embedding videos on the mobile app. It basically "proxies" the Referrer and
+ * allows any YouTube video to be embedded anywhere without restrictions.
+ */
+function serveYouTubeEmbed(url: string) {
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <meta name="referrer" content="strict-origin-when-cross-origin">
+    <meta name="robots" content="noindex,nofollow">
+    <title>YouTube Video Embed</title>
+    <style>
+    * {
+        margin: 0;
+        padding: 0;
+        box-sizing:border-box
+    }
+
+    body, html {
+        overflow: hidden;
+        background:#000
+    }
+
+    iframe {
+        border: 0;
+        width: 100vw;
+        height: 100vh;
+        display: block
+    }
+    </style>
+</head>
+<body>
+    <iframe src="${transformYouTubeUrl(
+      url
+    )}" allow="accelerometer;autoplay;clipboard-write;encrypted-media;gyroscope;picture-in-picture;web-share" allowfullscreen referrerpolicy="strict-origin-when-cross-origin" title="Video player"></iframe>
+</body>
+</html>`;
+}
+
+// Check if URL is a YouTube embed (including youtube-nocookie.com)
+function isYouTubeEmbed(urlString: string) {
+  const url = new URL(urlString);
+  return (
+    (url.hostname === "www.youtube.com" ||
+      url.hostname === "youtube.com" ||
+      url.hostname === "m.youtube.com" ||
+      url.hostname === "www.youtube-nocookie.com" ||
+      url.hostname === "youtube-nocookie.com") &&
+    url.pathname.startsWith("/embed/")
+  );
+}
+
+// Transform YouTube URLs to use youtube-nocookie.com for enhanced privacy
+function transformYouTubeUrl(urlString: string): string {
+  try {
+    const url = new URL(urlString);
+
+    // Check if it's a YouTube domain
+    if (
+      url.hostname === "www.youtube.com" ||
+      url.hostname === "youtube.com" ||
+      url.hostname === "m.youtube.com"
+    ) {
+      // Replace with youtube-nocookie.com
+      url.hostname = "www.youtube-nocookie.com";
+      return url.toString();
+    }
+
+    return urlString;
+  } catch {
+    return urlString;
+  }
+}