CalvinBackup/notesnook-sync-server_customPrivateV1
1
0
Fork 0
mirror of https://github.com/streetwriters/notesnook-sync-server.git synced 2026-02-12 11:12:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

sync: replace mongodb with file system based repository

Browse Source
This commit is contained in:
Abdullah Atta
2023-04-06 01:57:39 +05:00
parent 6f47574556
commit 8d20a9cff0
3 changed files with 76 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Notesnook.API/Hubs/SyncHub.cs
View File

@@ -87,10 +87,11 @@ namespace Notesnook.API.Hubs
long dateSynced = transferItem.LastSynced > userSettings.LastSynced ? transferItem.LastSynced : userSettings.LastSynced;
for (int i = 0; i < transferItem.Items.Length; ++i)
Parallel.For(0, transferItem.Items.Length, async (i) =>
{
var data = transferItem.Items[i];
var type = transferItem.Types[i];
var id = transferItem.Ids[i];
// We intentionally don't await here to speed up the sync. Fire and forget
// suits here because we don't really care if the item reaches the other
@@ -108,44 +109,39 @@ namespace Notesnook.API.Hubs
switch (type)
{
case "content":
Repositories.Contents.Upsert(JsonSerializer.Deserialize<Content>(data), userId, dateSynced);
await Repositories.Contents.UpsertAsync(id, data, userId, dateSynced);
break;
case "attachment":
Repositories.Attachments.Upsert(JsonSerializer.Deserialize<Attachment>(data), userId, dateSynced);
await Repositories.Attachments.UpsertAsync(id, data, userId, dateSynced);
break;
case "note":
Repositories.Notes.Upsert(JsonSerializer.Deserialize<Note>(data), userId, dateSynced);
await Repositories.Notes.UpsertAsync(id, data, userId, dateSynced);
break;
case "notebook":
Repositories.Notebooks.Upsert(JsonSerializer.Deserialize<Notebook>(data), userId, dateSynced);
await Repositories.Notebooks.UpsertAsync(id, data, userId, dateSynced);
break;
case "shortcut":
Repositories.Shortcuts.Upsert(JsonSerializer.Deserialize<Shortcut>(data), userId, dateSynced);
await Repositories.Shortcuts.UpsertAsync(id, data, userId, dateSynced);
break;
case "reminder":
Repositories.Reminders.Upsert(JsonSerializer.Deserialize<Reminder>(data), userId, dateSynced);
await Repositories.Reminders.UpsertAsync(id, data, userId, dateSynced);
break;
case "relation":
Repositories.Relations.Upsert(JsonSerializer.Deserialize<Relation>(data), userId, dateSynced);
await Repositories.Relations.UpsertAsync(id, data, userId, dateSynced);
break;
case "settings":
var settings = JsonSerializer.Deserialize<Setting>(data);
settings.Id = MongoDB.Bson.ObjectId.Parse(userId);
settings.ItemId = userId;
Repositories.Settings.Upsert(settings, userId, dateSynced);
await Repositories.Settings.UpsertAsync(userId, data, userId, dateSynced);
break;
case "vaultKey":
userSettings.VaultKey = JsonSerializer.Deserialize<EncryptedData>(data);
Repositories.UsersSettings.Upsert(userSettings, (u) => u.UserId == userId);
await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
break;
default:
throw new HubException("Invalid item type.");
}
});
}
return await unit.Commit() ? 1 : 0;
return 1;
}
public async Task<bool> SyncCompleted(long dateSynced)
@@ -262,6 +258,8 @@ namespace Notesnook.API.Hubs
[MessagePack.Key("types")]
public string[] Types { get; set; }
[MessagePack.Key("ids")]
public string[] Ids { get; set; }
[MessagePack.Key("total")]
public int Total { get; set; }

9
Notesnook.API/Models/Algorithms.cs
View File

@@ -17,10 +17,17 @@ You should have received a copy of the Affero GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
using System.Collections.Generic;
namespace Notesnook.API.Models
{
public class Algorithms
{
public static string Default => "xcha-argon2i13-7";
public const string Default = "xcha-argon2i13-7";
static readonly List<string> ALGORITHMS = new List<string> { Algorithms.Default };
public static bool IsValidAlgorithm(string algorithm)
{
return ALGORITHMS.Contains(algorithm);
}
}
}

114
Notesnook.API/Repositories/SyncItemsRepository.cs
View File

@@ -19,90 +19,82 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualBasic;
using MongoDB.Bson;
using MongoDB.Driver;
using Notesnook.API.Interfaces;
using Notesnook.API.Models;
using Streetwriters.Common;
using Streetwriters.Data.Interfaces;
using Streetwriters.Data.Repositories;
using Streetwriters.Data.Attributes;
namespace Notesnook.API.Repositories
{
public class SyncItemsRepository<T> : Repository<T> where T : SyncItem
public class SyncItemsRepository<T> where T : SyncItem
{
public SyncItemsRepository(IDbContext dbContext) : base(dbContext)
const string BASE_DATA_DIR = "data";
private string GetCollectionName()
{
Collection.Indexes.CreateOne(new CreateIndexModel<T>(Builders<T>.IndexKeys.Ascending(i => i.UserId).Descending(i => i.DateSynced)));
Collection.Indexes.CreateOne(new CreateIndexModel<T>(Builders<T>.IndexKeys.Ascending(i => i.UserId).Ascending((i) => i.ItemId)));
Collection.Indexes.CreateOne(new CreateIndexModel<T>(Builders<T>.IndexKeys.Ascending(i => i.UserId)));
var attribute = (BsonCollectionAttribute)typeof(T).GetCustomAttributes(
typeof(BsonCollectionAttribute),
true).FirstOrDefault();
if (string.IsNullOrEmpty(attribute.CollectionName) || string.IsNullOrEmpty(attribute.DatabaseName)) throw new Exception("Could not get a valid collection or database name.");
return attribute.CollectionName;
}
private readonly List<string> ALGORITHMS = new List<string> { Algorithms.Default };
private bool IsValidAlgorithm(string algorithm)
private string GetUserDirectoryPath(string userId)
{
return ALGORITHMS.Contains(algorithm);
return System.IO.Path.Join(BASE_DATA_DIR, userId, GetCollectionName());
}
public async Task<IEnumerable<T>> GetItemsSyncedAfterAsync(string userId, long timestamp)
private IEnumerable<string> EnumerateItems(string userId, string searchPattern = "*")
{
var cursor = await Collection.FindAsync(n => (n.DateSynced > timestamp) && n.UserId.Equals(userId));
return cursor.ToList();
try
{
return System.IO.Directory.EnumerateFiles(GetUserDirectoryPath(userId), searchPattern, System.IO.SearchOption.TopDirectoryOnly);
}
catch
{
return new string[] { };
}
}
// public async Task DeleteIdsAsync(string[] ids, string userId, CancellationToken token = default(CancellationToken))
// {
// await Collection.DeleteManyAsync<T>((i) => ids.Contains(i.Id) && i.UserId == userId, token);
// }
private string FindItemById(string userId, string id)
{
try
{
var files = Directory.GetFiles(GetUserDirectoryPath(userId), $"{id}-*", System.IO.SearchOption.TopDirectoryOnly);
return files.Length > 0 ? files[0] : null;
}
catch
{
return null;
}
}
public async Task<IEnumerable<string>> GetItemsSyncedAfterAsync(string userId, long timestamp)
{
var items = new List<string>();
await Parallel.ForEachAsync(EnumerateItems(userId), async (file, ct) =>
{
var parts = file.Split("-");
var id = parts[0];
var dateSynced = long.Parse(parts[1]);
if (dateSynced > timestamp) items.Add(await File.ReadAllTextAsync(file));
});
return items;
}
public void DeleteByUserId(string userId)
{
dbContext.AddCommand((handle, ct) => Collection.DeleteManyAsync<T>(handle, (i) => i.UserId == userId, cancellationToken: ct));
Directory.Delete(GetUserDirectoryPath(userId), true);
}
public async Task UpsertAsync(T item, string userId, long dateSynced)
public async Task UpsertAsync(string id, string item, string userId, long dateSynced)
{
if (item.Length > 15 * 1024 * 1024)
{
throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
}
if (!IsValidAlgorithm(item.Algorithm))
{
throw new Exception($"Invalid alg identifier {item.Algorithm}");
}
item.DateSynced = dateSynced;
item.UserId = userId;
await base.UpsertAsync(item, (x) => (x.ItemId == item.ItemId) && x.UserId == userId);
}
public void Upsert(T item, string userId, long dateSynced)
{
if (item.Length > 15 * 1024 * 1024)
{
throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
}
if (!IsValidAlgorithm(item.Algorithm))
{
throw new Exception($"Invalid alg identifier {item.Algorithm}");
}
item.DateSynced = dateSynced;
item.UserId = userId;
// await base.UpsertAsync(item, (x) => (x.ItemId == item.ItemId) && x.UserId == userId);
base.Upsert(item, (x) => (x.ItemId == item.ItemId) && x.UserId == userId);
Directory.CreateDirectory(GetUserDirectoryPath(userId));
var oldPath = FindItemById(userId, id);
var newPath = Path.Join(GetUserDirectoryPath(userId), $"{id}-{dateSynced}");
await File.WriteAllTextAsync(newPath, item);
if (oldPath != null) File.Delete(oldPath);
}
}
}