mirror of
https://github.com/streetwriters/notesnook-sync-server.git
synced 2026-11-02 14:23:57 +01:00
Abdullah Atta
ef857fec4f
In a versioned bucket, HEAD on a key whose latest version is a delete marker returns 405 MethodNotAllowed instead of 404. GetObjectSizeAsync threw on it, so re-adding a previously deleted attachment (same hash, same key) failed with a 500 before the upload started.
189 lines
7.4 KiB
C#
189 lines
7.4 KiB
C#
using Amazon.S3.Model;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using MongoDB.Bson;
|
|
using MongoDB.Driver;
|
|
using Notesnook.API.Models;
|
|
using Notesnook.API.Services;
|
|
using Streetwriters.Common;
|
|
using Streetwriters.Common.Accessors;
|
|
using Streetwriters.Common.Enums;
|
|
using Streetwriters.Common.Exceptions;
|
|
using Streetwriters.Common.Interfaces;
|
|
using Streetwriters.Common.Models;
|
|
|
|
namespace Notesnook.API.Tests;
|
|
|
|
public class S3ServiceTests(TestDatabase db, MinioFixture minio) : IClassFixture<TestDatabase>, IClassFixture<MinioFixture>
|
|
{
|
|
private const long MB = 1024 * 1024;
|
|
private static readonly HttpClient Http = new();
|
|
|
|
private sealed class HttpClients : IHttpClientFactory
|
|
{
|
|
public HttpClient CreateClient(string name) => new();
|
|
}
|
|
|
|
private static IUserSubscriptionService FreePlan(string userId) => Stub<IUserSubscriptionService>.Create(
|
|
("GetUserSubscriptionAsync", (_) => Task.FromResult<Subscription?>(new Subscription { UserId = userId, AppId = ApplicationType.NOTESNOOK, Plan = SubscriptionPlan.FREE })));
|
|
|
|
private S3Service CreateService(IUserSubscriptionService? subscriptions = null)
|
|
{
|
|
_ = minio;
|
|
var accessor = new WampServiceAccessor(Servers.NotesnookAPI) { UserSubscriptionService = subscriptions };
|
|
return new S3Service(new S3Clients(NullLogger<S3Service>.Instance), new HttpClients(), db.CreateScope().Repositories, accessor, NullLogger<S3Service>.Instance);
|
|
}
|
|
|
|
private static string NewUserId() => ObjectId.GenerateNewId().ToString();
|
|
|
|
private static async Task UploadAsync(S3Service s3, string userId, string name, byte[] data)
|
|
{
|
|
var url = await s3.GetInternalUploadObjectUrlAsync(userId, name);
|
|
using var response = await Http.PutAsync(url, new ByteArrayContent(data));
|
|
response.EnsureSuccessStatusCode();
|
|
}
|
|
|
|
private async Task CreateSettingsAsync(string userId, long used)
|
|
{
|
|
await db.CreateScope().Repositories.UsersSettings.Collection.InsertOneAsync(new UserSettings
|
|
{
|
|
UserId = userId,
|
|
Salt = "",
|
|
StorageLimit = new Limit { Value = used, UpdatedAt = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() }
|
|
});
|
|
}
|
|
|
|
private async Task<long> UsedAsync(string userId) =>
|
|
(await db.CreateScope().Repositories.UsersSettings.Collection.Find((u) => u.UserId == userId).FirstAsync()).StorageLimit!.Value;
|
|
|
|
private async Task<CompleteMultipartUploadRequest> UploadPartAsync(S3Service s3, string userId, string name, byte[] data)
|
|
{
|
|
var meta = await s3.StartMultipartUploadAsync(userId, name, 1);
|
|
using var response = await Http.PutAsync(meta.Parts[0], new ByteArrayContent(data));
|
|
response.EnsureSuccessStatusCode();
|
|
return new CompleteMultipartUploadRequest
|
|
{
|
|
Key = name,
|
|
UploadId = meta.UploadId,
|
|
PartETags = [new PartETag(1, response.Headers.ETag!.Tag)]
|
|
};
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Uploaded_objects_report_their_size()
|
|
{
|
|
var s3 = CreateService();
|
|
var userId = NewUserId();
|
|
|
|
await UploadAsync(s3, userId, "file", new byte[5]);
|
|
|
|
Assert.Equal(5, await s3.GetObjectSizeAsync(userId, "file"));
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(userId, "missing"));
|
|
}
|
|
|
|
private sealed class StatusHandler(System.Net.HttpStatusCode status) : HttpMessageHandler
|
|
{
|
|
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
|
|
Task.FromResult(new HttpResponseMessage(status));
|
|
}
|
|
|
|
private sealed class StatusHttpClients(System.Net.HttpStatusCode status) : IHttpClientFactory
|
|
{
|
|
public HttpClient CreateClient(string name) => new(new StatusHandler(status));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deleted_objects_in_a_versioned_bucket_report_no_size()
|
|
{
|
|
_ = minio;
|
|
var accessor = new WampServiceAccessor(Servers.NotesnookAPI);
|
|
var s3 = new S3Service(new S3Clients(NullLogger<S3Service>.Instance), new StatusHttpClients(System.Net.HttpStatusCode.MethodNotAllowed), db.CreateScope().Repositories, accessor, NullLogger<S3Service>.Instance);
|
|
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(NewUserId(), "deleted"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deleting_an_object_removes_it()
|
|
{
|
|
var s3 = CreateService();
|
|
var userId = NewUserId();
|
|
await UploadAsync(s3, userId, "file", new byte[5]);
|
|
|
|
await s3.DeleteObjectAsync(userId, "file");
|
|
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(userId, "file"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deleting_a_directory_removes_only_that_users_objects()
|
|
{
|
|
var s3 = CreateService();
|
|
var userId = NewUserId();
|
|
var otherUserId = NewUserId();
|
|
foreach (var name in new[] { "a", "b", "c" }) await UploadAsync(s3, userId, name, new byte[5]);
|
|
await UploadAsync(s3, otherUserId, "a", new byte[5]);
|
|
|
|
await s3.DeleteDirectoryAsync(userId);
|
|
|
|
foreach (var name in new[] { "a", "b", "c" }) Assert.Equal(0, await s3.GetObjectSizeAsync(userId, name));
|
|
Assert.Equal(5, await s3.GetObjectSizeAsync(otherUserId, "a"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deleting_a_directory_ignores_users_whose_id_starts_with_the_same_characters()
|
|
{
|
|
var s3 = CreateService();
|
|
var userId = NewUserId();
|
|
var otherUserId = userId + "0";
|
|
await UploadAsync(s3, userId, "a", new byte[5]);
|
|
await UploadAsync(s3, otherUserId, "a", new byte[5]);
|
|
|
|
await s3.DeleteDirectoryAsync(userId);
|
|
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(userId, "a"));
|
|
Assert.Equal(5, await s3.GetObjectSizeAsync(otherUserId, "a"));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Deleting_a_directory_removes_more_objects_than_fit_in_one_request()
|
|
{
|
|
var s3 = CreateService();
|
|
var userId = NewUserId();
|
|
// S3 deletes at most 1000 keys per request
|
|
var names = Enumerable.Range(0, 1001).Select((i) => $"file{i}").ToArray();
|
|
await Parallel.ForEachAsync(names, new ParallelOptions { MaxDegreeOfParallelism = 32 }, async (name, _) => await UploadAsync(s3, userId, name, new byte[1]));
|
|
|
|
await s3.DeleteDirectoryAsync(userId);
|
|
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(userId, names[0]));
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(userId, names[^1]));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_multipart_upload_within_the_limit_is_completed_and_charged()
|
|
{
|
|
var userId = NewUserId();
|
|
var s3 = CreateService(FreePlan(userId));
|
|
await CreateSettingsAsync(userId, used: 0);
|
|
|
|
await s3.CompleteMultipartUploadAsync(userId, await UploadPartAsync(s3, userId, "file", new byte[5]));
|
|
|
|
Assert.Equal(5, await s3.GetObjectSizeAsync(userId, "file"));
|
|
Assert.Equal(5, await UsedAsync(userId));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task A_multipart_upload_past_the_limit_is_aborted_and_not_charged()
|
|
{
|
|
var userId = NewUserId();
|
|
var s3 = CreateService(FreePlan(userId));
|
|
// the free plan allows 50 MB a month
|
|
await CreateSettingsAsync(userId, used: 50 * MB - 3);
|
|
|
|
var error = await Assert.ThrowsAsync<ClientException>(async () => await s3.CompleteMultipartUploadAsync(userId, await UploadPartAsync(s3, userId, "file", new byte[5])));
|
|
|
|
Assert.Equal("Storage limit reached.", error.Message);
|
|
Assert.Equal(0, await s3.GetObjectSizeAsync(userId, "file"));
|
|
Assert.Equal(50 * MB - 3, await UsedAsync(userId));
|
|
}
|
|
}
|