CalvinBackup/notesnook-sync-server_customPrivateV1
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook-sync-server.git synced 2026-02-12 19:22:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

sync: refactor sync device service to be more memory efficient

Browse Source
This commit is contained in:
Abdullah Atta
2024-06-07 10:55:15 +05:00
parent 7ce02d0193
commit 0c0ade0c64
4 changed files with 127 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Notesnook.API/Controllers/SyncDeviceController.cs
View File

@@ -44,7 +44,7 @@ namespace Notesnook.API.Controllers
try
{
var userId = this.User.FindFirstValue("sub");
SyncDeviceService.RegisterDevice(userId, deviceId);
new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).RegisterDevice();
return Ok();
}
catch (Exception ex)
@@ -61,7 +61,7 @@ namespace Notesnook.API.Controllers
try
{
var userId = this.User.FindFirstValue("sub");
SyncDeviceService.UnregisterDevice(userId, deviceId);
new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).UnregisterDevice();
return Ok();
}
catch (Exception ex)

19
Notesnook.API/Hubs/SyncV2Hub.cs
View File

@@ -131,8 +131,8 @@ namespace Notesnook.API.Hubs
if (!await unit.Commit()) return 0;
await SyncDeviceService.AddIdsToOtherDevicesAsync(userId, deviceId, pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
return 1;
await new SyncDeviceService(new SyncDevice(ref userId, ref deviceId)).AddIdsToOtherDevicesAsync(pushItem.Items.Select((i) => $"{i.ItemId}:{pushItem.Type}").ToList());
return 1;
}
public async Task<bool> PushCompleted()
@@ -193,13 +193,12 @@ namespace Notesnook.API.Hubs
public async Task<SyncV2Metadata> RequestFetch(string deviceId)
{
var userId = Context.User.FindFirstValue("sub");
var deviceService = new SyncDeviceService(new SyncDevice(ref userId, ref deviceId));
if (!deviceService.IsDeviceRegistered()) deviceService.RegisterDevice();
if (!SyncDeviceService.IsDeviceRegistered(userId, deviceId))
SyncDeviceService.RegisterDevice(userId, deviceId);
var isResetSync = SyncDeviceService.IsSyncReset(userId, deviceId);
if (!SyncDeviceService.IsUnsynced(userId, deviceId) &&
!SyncDeviceService.IsSyncPending(userId, deviceId) &&
var isResetSync = deviceService.IsSyncReset();
if (!deviceService.IsUnsynced() &&
!deviceService.IsSyncPending() &&
!isResetSync)
return new SyncV2Metadata { Synced = true };
@@ -241,11 +240,11 @@ namespace Notesnook.API.Hubs
{
var syncedIds = chunk.Items.Select((i) => $"{i.ItemId}:{chunk.Type}").ToHashSet();
ids = ids.Where((id) => !syncedIds.Contains(id)).ToArray();
await SyncDeviceService.WritePendingIdsAsync(userId, deviceId, ids);
await deviceService.WritePendingIdsAsync(ids);
}
}
SyncDeviceService.Reset(userId, deviceId);
deviceService.Reset();
return new SyncV2Metadata
{

173
Notesnook.API/Services/SyncDeviceService.cs
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -26,46 +27,94 @@ using System.Threading.Tasks;
namespace Notesnook.API.Services
{
public class SyncDeviceService
public struct SyncDevice(ref string userId, ref string deviceId)
{
private static string UserSyncDirectoryPath(string userId) => Path.Join("sync", userId);
private static string UserDeviceDirectoryPath(string userId, string deviceId) => Path.Join(SyncDeviceService.UserSyncDirectoryPath(userId), deviceId);
public readonly string DeviceId = deviceId;
public readonly string UserId = userId;
private static string PendingIdsFilePath(string userId, string deviceId) => Path.Join(SyncDeviceService.UserDeviceDirectoryPath(userId, deviceId), "pending");
private static string UnsyncedIdsFilePath(string userId, string deviceId) => Path.Join(SyncDeviceService.UserDeviceDirectoryPath(userId, deviceId), "unsynced");
private static string ResetSyncFilePath(string userId, string deviceId) => Path.Join(SyncDeviceService.UserDeviceDirectoryPath(userId, deviceId), "reset-sync");
public static async Task<string[]> GetUnsyncedIdsAsync(string userId, string deviceId)
private string userSyncDirectoryPath = null;
public string UserSyncDirectoryPath
{
get
{
userSyncDirectoryPath ??= Path.Join("sync", UserId);
return userSyncDirectoryPath;
}
}
private string userDeviceDirectoryPath = null;
public string UserDeviceDirectoryPath
{
get
{
userDeviceDirectoryPath ??= Path.Join(UserSyncDirectoryPath, DeviceId);
return userDeviceDirectoryPath;
}
}
private string pendingIdsFilePath = null;
public string PendingIdsFilePath
{
get
{
pendingIdsFilePath ??= Path.Join(UserDeviceDirectoryPath, "pending");
return pendingIdsFilePath;
}
}
private string unsyncedIdsFilePath = null;
public string UnsyncedIdsFilePath
{
get
{
unsyncedIdsFilePath ??= Path.Join(UserDeviceDirectoryPath, "unsynced");
return unsyncedIdsFilePath;
}
}
private string resetSyncFilePath = null;
public string ResetSyncFilePath
{
get
{
resetSyncFilePath ??= Path.Join(UserDeviceDirectoryPath, "reset-sync");
return resetSyncFilePath;
}
}
}
public class SyncDeviceService(SyncDevice device)
{
public async Task<string[]> GetUnsyncedIdsAsync()
{
try
{
return await File.ReadAllLinesAsync(UnsyncedIdsFilePath(userId, deviceId));
}
catch
{
return Array.Empty<string>();
return await File.ReadAllLinesAsync(device.UnsyncedIdsFilePath);
}
catch { return []; }
}
public static async Task<string[]> FetchUnsyncedIdsAsync(string userId, string deviceId)
public async Task<string[]> GetUnsyncedIdsAsync(string deviceId)
{
if (IsSyncReset(userId, deviceId)) return Array.Empty<string>();
if (UnsyncedIdsFileLocks.TryGetValue(deviceId, out SemaphoreSlim fileLock) && fileLock.CurrentCount == 0)
try
{
return await File.ReadAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, deviceId, "unsynced"));
}
catch { return []; }
}
public async Task<string[]> FetchUnsyncedIdsAsync()
{
if (IsSyncReset()) return Array.Empty<string>();
if (UnsyncedIdsFileLocks.TryGetValue(device.DeviceId, out SemaphoreSlim fileLock) && fileLock.CurrentCount == 0)
await fileLock.WaitAsync();
try
{
var unsyncedIds = await GetUnsyncedIdsAsync(userId, deviceId);
if (IsSyncPending(userId, deviceId))
var unsyncedIds = await GetUnsyncedIdsAsync();
if (IsSyncPending())
{
unsyncedIds = unsyncedIds.Union(await File.ReadAllLinesAsync(PendingIdsFilePath(userId, deviceId))).ToArray();
unsyncedIds = unsyncedIds.Union(await File.ReadAllLinesAsync(device.PendingIdsFilePath)).ToArray();
}
if (unsyncedIds.Length == 0) return Array.Empty<string>();
if (unsyncedIds.Length == 0) return [];
File.Delete(UnsyncedIdsFilePath(userId, deviceId));
await File.WriteAllLinesAsync(PendingIdsFilePath(userId, deviceId), unsyncedIds);
File.Delete(device.UnsyncedIdsFilePath);
await File.WriteAllLinesAsync(device.PendingIdsFilePath, unsyncedIds);
return unsyncedIds;
}
@@ -80,87 +129,93 @@ namespace Notesnook.API.Services
}
public static async Task WritePendingIdsAsync(string userId, string deviceId, IEnumerable<string> ids)
public async Task WritePendingIdsAsync(IEnumerable<string> ids)
{
await File.WriteAllLinesAsync(PendingIdsFilePath(userId, deviceId), ids);
await File.WriteAllLinesAsync(device.PendingIdsFilePath, ids);
}
public static bool IsSyncReset(string userId, string deviceId)
public bool IsSyncReset()
{
return File.Exists(ResetSyncFilePath(userId, deviceId));
return File.Exists(device.ResetSyncFilePath);
}
public bool IsSyncReset(string deviceId)
{
return File.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId, "reset-sync"));
}
public static bool IsSyncPending(string userId, string deviceId)
public bool IsSyncPending()
{
return File.Exists(PendingIdsFilePath(userId, deviceId));
return File.Exists(device.PendingIdsFilePath);
}
public static bool IsUnsynced(string userId, string deviceId)
public bool IsUnsynced()
{
return File.Exists(UnsyncedIdsFilePath(userId, deviceId));
return File.Exists(device.UnsyncedIdsFilePath);
}
public static void Reset(string userId, string deviceId)
public void Reset()
{
File.Delete(ResetSyncFilePath(userId, deviceId));
File.Delete(PendingIdsFilePath(userId, deviceId));
File.Delete(device.ResetSyncFilePath);
File.Delete(device.PendingIdsFilePath);
}
public static bool IsDeviceRegistered(string userId, string deviceId)
public bool IsDeviceRegistered()
{
return Directory.Exists(UserDeviceDirectoryPath(userId, deviceId));
return Directory.Exists(device.UserDeviceDirectoryPath);
}
public bool IsDeviceRegistered(string deviceId)
{
return Directory.Exists(Path.Join(device.UserSyncDirectoryPath, deviceId));
}
public static IEnumerable<string> ListDevices(string userId)
public string[] ListDevices()
{
return Directory.EnumerateDirectories(UserSyncDirectoryPath(userId)).Select((path) => Path.GetFileName(path));
return Directory.GetDirectories(device.UserSyncDirectoryPath).Select((path) => path[(path.LastIndexOf(Path.DirectorySeparatorChar) + 1)..]).ToArray();
}
public static void ResetDevices(string userId)
public void ResetDevices()
{
if (File.Exists(UserSyncDirectoryPath(userId))) File.Delete(UserSyncDirectoryPath(userId));
Directory.CreateDirectory(UserSyncDirectoryPath(userId));
if (File.Exists(device.UserSyncDirectoryPath)) File.Delete(device.UserSyncDirectoryPath);
Directory.CreateDirectory(device.UserSyncDirectoryPath);
}
private static readonly Dictionary<string, SemaphoreSlim> UnsyncedIdsFileLocks = new();
public static async Task AddIdsToOtherDevicesAsync(string userId, string deviceId, List<string> ids)
private readonly ConcurrentDictionary<string, SemaphoreSlim> UnsyncedIdsFileLocks = [];
public async Task AddIdsToOtherDevicesAsync(List<string> ids)
{
foreach (var id in ListDevices(userId))
await Parallel.ForEachAsync(ListDevices(), async (id, ct) =>
{
if (id == deviceId || IsSyncReset(userId, id)) continue;
if (id == device.DeviceId || IsSyncReset(id)) return;
if (!UnsyncedIdsFileLocks.TryGetValue(id, out SemaphoreSlim fileLock))
{
fileLock = new SemaphoreSlim(1, 1);
UnsyncedIdsFileLocks.Add(id, fileLock);
fileLock = UnsyncedIdsFileLocks.AddOrUpdate(id, (id) => new SemaphoreSlim(1, 1), (id, old) => new SemaphoreSlim(1, 1));
}
await fileLock.WaitAsync();
await fileLock.WaitAsync(ct);
try
{
if (!IsDeviceRegistered(userId, id)) Directory.CreateDirectory(UserDeviceDirectoryPath(userId, id));
if (!IsDeviceRegistered(id)) Directory.CreateDirectory(Path.Join(device.UserSyncDirectoryPath, id));
var oldIds = await GetUnsyncedIdsAsync(userId, id);
await File.WriteAllLinesAsync(UnsyncedIdsFilePath(userId, id), ids.Union(oldIds));
var oldIds = await GetUnsyncedIdsAsync(id);
await File.WriteAllLinesAsync(Path.Join(device.UserSyncDirectoryPath, id, "unsynced"), ids.Union(oldIds), ct);
}
finally
{
fileLock.Release();
}
}
});
}
public static void RegisterDevice(string userId, string deviceId)
public void RegisterDevice()
{
Directory.CreateDirectory(UserDeviceDirectoryPath(userId, deviceId));
File.Create(ResetSyncFilePath(userId, deviceId)).Close();
Directory.CreateDirectory(device.UserDeviceDirectoryPath);
File.Create(device.ResetSyncFilePath).Close();
}
public static void UnregisterDevice(string userId, string deviceId)
public void UnregisterDevice()
{
try
{
Directory.Delete(UserDeviceDirectoryPath(userId, deviceId), true);
Directory.Delete(device.UserDeviceDirectoryPath, true);
}
catch { }
}

4
Notesnook.API/Services/UserService.cs
View File

@@ -172,7 +172,7 @@ namespace Notesnook.API.Services
{
await Slogger<UserService>.Info(nameof(DeleteUserAsync), "Deleting user account", userId);
SyncDeviceService.ResetDevices(userId);
new SyncDeviceService(new SyncDevice(ref userId, ref userId)).ResetDevices();
var cc = new CancellationTokenSource();
@@ -228,7 +228,7 @@ namespace Notesnook.API.Services
public async Task<bool> ResetUserAsync(string userId, bool removeAttachments)
{
SyncDeviceService.ResetDevices(userId);
new SyncDeviceService(new SyncDevice(ref userId, ref userId)).ResetDevices();
var cc = new CancellationTokenSource();