CalvinBackup/notesnook-sync-server_customPrivateV1
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook-sync-server.git synced 2026-02-12 19:22:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

notesnook: add opentelemetry based metric logging

Browse Source
This commit is contained in:
Abdullah Atta
2024-06-07 15:41:44 +05:00
parent b304d314a0
commit 59cf7ffcde
6 changed files with 133 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
Notesnook.API/EventSources/SyncEventCounterSource.cs Normal file
View File

@@ -0,0 +1,49 @@
using System;
using System.Diagnostics.Metrics;
using System.Diagnostics.Tracing;
[EventSource(Name = "Notesnook.API.EventCounter.Sync")]
public sealed class SyncEventCounterSource : EventSource
{
public static readonly SyncEventCounterSource Log = new();
private Meter meter = new("Notesnook.API.Metrics.Sync", "1.0.0");
private Counter<int> fetchCounter;
private Counter<int> pushCounter;
private Counter<int> legacyFetchCounter;
private Counter<int> pushV2Counter;
private Counter<int> fetchV2Counter;
private Histogram<long> fetchV2Duration;
private Histogram<long> pushV2Duration;
private SyncEventCounterSource()
{
fetchCounter = meter.CreateCounter<int>("sync.fetches", "fetches", "Total fetches");
pushCounter = meter.CreateCounter<int>("sync.pushes", "pushes", "Total pushes");
legacyFetchCounter = meter.CreateCounter<int>("sync.legacy-fetches", "fetches", "Total legacy fetches");
fetchV2Counter = meter.CreateCounter<int>("sync.v2.fetches", "fetches", "Total v2 fetches");
pushV2Counter = meter.CreateCounter<int>("sync.v2.pushes", "pushes", "Total v2 pushes");
fetchV2Duration = meter.CreateHistogram<long>("sync.v2.fetch_duration");
pushV2Duration = meter.CreateHistogram<long>("sync.v2.push_duration");
}
public void Fetch() => fetchCounter.Add(1);
public void LegacyFetch() => legacyFetchCounter.Add(1);
public void FetchV2() => fetchV2Counter.Add(1);
public void PushV2() => pushV2Counter.Add(1);
public void Push() => pushCounter.Add(1);
public void RecordFetchDuration(long durationMs) => fetchV2Duration.Record(durationMs);
public void RecordPushDuration(long durationMs) => pushV2Duration.Record(durationMs);
protected override void Dispose(bool disposing)
{
legacyFetchCounter = null;
fetchV2Counter = null;
pushV2Counter = null;
pushCounter = null;
fetchCounter = null;
meter.Dispose();
meter = null;
base.Dispose(disposing);
}
}

4
Notesnook.API/Hubs/SyncHub.cs
View File

@@ -230,6 +230,8 @@ namespace Notesnook.API.Hubs
var userId = Context.User.FindFirstValue("sub");
if (string.IsNullOrEmpty(userId)) return 0;
SyncEventCounterSource.Log.Push();
try
{
var others = Clients.OthersInGroup(userId);
@@ -347,6 +349,8 @@ namespace Notesnook.API.Hubs
throw new HubException("Cannot fetch data while another sync is in progress. Please try again later.");
}
SyncEventCounterSource.Log.Fetch();
var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId == userId);
if (userSettings.LastSynced > 0 && lastSyncedTimestamp > userSettings.LastSynced)
{

100
Notesnook.API/Hubs/SyncV2Hub.cs
View File

@@ -19,6 +19,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Security.Claims;
using System.Text.Json.Serialization;
@@ -26,6 +28,8 @@ using System.Threading.Tasks;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.SignalR;
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Driver;
using Notesnook.API.Authorization;
using Notesnook.API.Interfaces;
@@ -121,18 +125,28 @@ namespace Notesnook.API.Hubs
public async Task<int> PushItems(string deviceId, SyncTransferItemV2 pushItem)
{
var userId = Context.User.FindFirstValue("sub");
if (string.IsNullOrEmpty(userId)) return 0;
if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
var UpsertItem = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception($"Invalid item type: {pushItem.Type}.");
foreach (var item in pushItem.Items)
SyncEventCounterSource.Log.PushV2();
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
UpsertItem(item, userId, 1);
}
if (!await unit.Commit()) return 0;
var UpsertItems = MapTypeToUpsertAction(pushItem.Type) ?? throw new Exception($"Invalid item type: {pushItem.Type}.");
UpsertItems(pushItem.Items, userId, 1);
if (!await unit.Commit()) return 0;
await new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).AddIdsToOtherDevicesAsync(pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
return 1;
}
finally
{
stopwatch.Stop();
SyncEventCounterSource.Log.RecordPushDuration(stopwatch.ElapsedMilliseconds);
}
}
public async Task<bool> PushCompleted()
@@ -193,6 +207,10 @@ namespace Notesnook.API.Hubs
public async Task<SyncV2Metadata> RequestFetch(string deviceId)
{
var userId = Context.User.FindFirstValue("sub");
if (string.IsNullOrEmpty(userId)) throw new HubException("Please login to sync.");
SyncEventCounterSource.Log.FetchV2();
var deviceService = new SyncDeviceService(new SyncDevice(ref userId, ref deviceId));
if (!deviceService.IsDeviceRegistered()) deviceService.RegisterDevice();
@@ -202,11 +220,15 @@ namespace Notesnook.API.Hubs
!isResetSync)
return new SyncV2Metadata { Synced = true };
string[] ids = await SyncDeviceService.FetchUnsyncedIdsAsync(userId, deviceId);
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
string[] ids = await deviceService.FetchUnsyncedIdsAsync();
var chunks = PrepareChunks(
collections: [
Repositories.Settings.FindItemsById,
var chunks = PrepareChunks(
collections: [
Repositories.Settings.FindItemsById,
Repositories.Attachments.FindItemsById,
Repositories.Notes.FindItemsById,
Repositories.Notebooks.FindItemsById,
@@ -217,39 +239,45 @@ namespace Notesnook.API.Hubs
Repositories.Tags.FindItemsById,
Repositories.Vaults.FindItemsById,
Repositories.Relations.FindItemsById,
],
types: CollectionKeys,
userId,
ids,
size: 1000,
resetSync: isResetSync,
maxBytes: 7 * 1024 * 1024
);
],
types: CollectionKeys,
userId,
ids,
size: 1000,
resetSync: isResetSync,
maxBytes: 7 * 1024 * 1024
);
var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId.Equals(userId));
if (userSettings.VaultKey != null)
{
if (!await Clients.Caller.SendVaultKey(userSettings.VaultKey).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected vault key.");
}
await foreach (var chunk in chunks)
{
if (!await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected sent items.");
if (!isResetSync)
var userSettings = await Repositories.UsersSettings.FindOneAsync((u) => u.UserId.Equals(userId));
if (userSettings.VaultKey != null)
{
var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
await deviceService.WritePendingIdsAsync(ids);
if (!await Clients.Caller.SendVaultKey(userSettings.VaultKey).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected vault key.");
}
await foreach (var chunk in chunks)
{
if (!await Clients.Caller.SendItems(chunk).WaitAsync(TimeSpan.FromMinutes(10))) throw new HubException("Client rejected sent items.");
if (!isResetSync)
{
var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
await deviceService.WritePendingIdsAsync(ids);
}
}
}
deviceService.Reset();
return new SyncV2Metadata
return new SyncV2Metadata
{
Synced = true,
};
}
finally
{
Synced = true,
};
stopwatch.Stop();
SyncEventCounterSource.Log.RecordFetchDuration(stopwatch.ElapsedMilliseconds);
}
}
}

2
Notesnook.API/Notesnook.API.csproj
View File

@@ -15,6 +15,8 @@
<PackageReference Include="AWSSDK.S3" Version="3.7.9.21" />
<PackageReference Include="Microsoft.AspNetCore.SignalR.Protocols.MessagePack" Version="6.0.3" />
<PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Https" Version="2.2.0" />
<PackageReference Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-alpha.2" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
</ItemGroup>
<ItemGroup>

3
Notesnook.API/Program.cs
View File

@@ -26,6 +26,8 @@ using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting;
using Streetwriters.Common;
using System.Linq;
using Microsoft.Extensions.Logging;
using System.Net;
namespace Notesnook.API
{
@@ -59,6 +61,7 @@ namespace Notesnook.API
listenerOptions.UseHttps(Servers.NotesnookAPI.SSLCertificate);
});
}
options.Listen(IPAddress.Parse("127.0.0.1"), 5067);
});
});
}

11
Notesnook.API/Startup.cs
View File

@@ -51,6 +51,8 @@ using Notesnook.API.Interfaces;
using Notesnook.API.Models;
using Notesnook.API.Repositories;
using Notesnook.API.Services;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using Streetwriters.Common;
using Streetwriters.Common.Extensions;
using Streetwriters.Common.Messages;
@@ -208,6 +210,13 @@ namespace Notesnook.API
{
options.Level = CompressionLevel.Fastest;
});
services.AddOpenTelemetry()
.ConfigureResource(resource => resource
.AddService(serviceName: "Notesnook.API"))
.WithMetrics((builder) => builder
.AddMeter("Notesnook.API.Metrics.Sync")
.AddPrometheusExporter());
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
@@ -221,6 +230,7 @@ namespace Notesnook.API
});
}
app.UseOpenTelemetryPrometheusScrapingEndpoint((context) => context.Request.Path == "/metrics" && context.Connection.LocalPort == 5067);
app.UseResponseCompression();
app.UseCors("notesnook");
@@ -248,6 +258,7 @@ namespace Notesnook.API
app.UseEndpoints(endpoints =>
{
endpoints.MapPrometheusScrapingEndpoint();
endpoints.MapControllers();
endpoints.MapHealthChecks("/health");
endpoints.MapHub<SyncHub>("/hubs/sync", options =>