sync: replace mongodb with file system based repository

Notesnook.API/Hubs/SyncHub.cs

@@ -87,10 +87,11 @@ namespace Notesnook.API.Hubs
 
             long dateSynced = transferItem.LastSynced > userSettings.LastSynced ? transferItem.LastSynced : userSettings.LastSynced;
 
-            for (int i = 0; i < transferItem.Items.Length; ++i)
+            Parallel.For(0, transferItem.Items.Length, async (i) =>
             {
                 var data = transferItem.Items[i];
                 var type = transferItem.Types[i];
+                var id = transferItem.Ids[i];
 
                 // We intentionally don't await here to speed up the sync. Fire and forget
                 // suits here because we don't really care if the item reaches the other
@@ -108,44 +109,39 @@ namespace Notesnook.API.Hubs
                 switch (type)
                 {
                     case "content":
-                        Repositories.Contents.Upsert(JsonSerializer.Deserialize<Content>(data), userId, dateSynced);
+                        await Repositories.Contents.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "attachment":
-                        Repositories.Attachments.Upsert(JsonSerializer.Deserialize<Attachment>(data), userId, dateSynced);
+                        await Repositories.Attachments.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "note":
-                        Repositories.Notes.Upsert(JsonSerializer.Deserialize<Note>(data), userId, dateSynced);
+                        await Repositories.Notes.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "notebook":
-                        Repositories.Notebooks.Upsert(JsonSerializer.Deserialize<Notebook>(data), userId, dateSynced);
+                        await Repositories.Notebooks.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "shortcut":
-                        Repositories.Shortcuts.Upsert(JsonSerializer.Deserialize<Shortcut>(data), userId, dateSynced);
+                        await Repositories.Shortcuts.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "reminder":
-                        Repositories.Reminders.Upsert(JsonSerializer.Deserialize<Reminder>(data), userId, dateSynced);
+                        await Repositories.Reminders.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "relation":
-                        Repositories.Relations.Upsert(JsonSerializer.Deserialize<Relation>(data), userId, dateSynced);
+                        await Repositories.Relations.UpsertAsync(id, data, userId, dateSynced);
                         break;
                     case "settings":
-                        var settings = JsonSerializer.Deserialize<Setting>(data);
+                        await Repositories.Settings.UpsertAsync(userId, data, userId, dateSynced);
-                        settings.Id = MongoDB.Bson.ObjectId.Parse(userId);
-                        settings.ItemId = userId;
-                        Repositories.Settings.Upsert(settings, userId, dateSynced);
                         break;
                     case "vaultKey":
                         userSettings.VaultKey = JsonSerializer.Deserialize<EncryptedData>(data);
-                        Repositories.UsersSettings.Upsert(userSettings, (u) => u.UserId == userId);
+                        await Repositories.UsersSettings.UpsertAsync(userSettings, (u) => u.UserId == userId);
                         break;
                     default:
                         throw new HubException("Invalid item type.");
                 }
+            });
 
-            }
+            return 1;
-
-            return await unit.Commit() ? 1 : 0;
-
         }
 
         public async Task<bool> SyncCompleted(long dateSynced)
@@ -262,6 +258,8 @@ namespace Notesnook.API.Hubs
 
         [MessagePack.Key("types")]
         public string[] Types { get; set; }
+        [MessagePack.Key("ids")]
+        public string[] Ids { get; set; }
 
         [MessagePack.Key("total")]
         public int Total { get; set; }

Notesnook.API/Models/Algorithms.cs

@@ -17,10 +17,17 @@ You should have received a copy of the Affero GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
+using System.Collections.Generic;
+
 namespace Notesnook.API.Models
 {
     public class Algorithms
     {
-        public static string Default => "xcha-argon2i13-7";
+        public const string Default = "xcha-argon2i13-7";
+        static readonly List<string> ALGORITHMS = new List<string> { Algorithms.Default };
+        public static bool IsValidAlgorithm(string algorithm)
+        {
+            return ALGORITHMS.Contains(algorithm);
+        }
     }
 }

Notesnook.API/Repositories/SyncItemsRepository.cs

@@ -19,90 +19,82 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
-using System.Security.Cryptography;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.VisualBasic;
-using MongoDB.Bson;
 using MongoDB.Driver;
-using Notesnook.API.Interfaces;
 using Notesnook.API.Models;
-using Streetwriters.Common;
+using Streetwriters.Data.Attributes;
-using Streetwriters.Data.Interfaces;
-using Streetwriters.Data.Repositories;
 
 namespace Notesnook.API.Repositories
 {
-    public class SyncItemsRepository<T> : Repository<T> where T : SyncItem
+    public class SyncItemsRepository<T> where T : SyncItem
     {
-        public SyncItemsRepository(IDbContext dbContext) : base(dbContext)
+        const string BASE_DATA_DIR = "data";
+        private string GetCollectionName()
         {
-            Collection.Indexes.CreateOne(new CreateIndexModel<T>(Builders<T>.IndexKeys.Ascending(i => i.UserId).Descending(i => i.DateSynced)));
+            var attribute = (BsonCollectionAttribute)typeof(T).GetCustomAttributes(
-            Collection.Indexes.CreateOne(new CreateIndexModel<T>(Builders<T>.IndexKeys.Ascending(i => i.UserId).Ascending((i) => i.ItemId)));
+                    typeof(BsonCollectionAttribute),
-            Collection.Indexes.CreateOne(new CreateIndexModel<T>(Builders<T>.IndexKeys.Ascending(i => i.UserId)));
+                    true).FirstOrDefault();
+            if (string.IsNullOrEmpty(attribute.CollectionName) || string.IsNullOrEmpty(attribute.DatabaseName)) throw new Exception("Could not get a valid collection or database name.");
+            return attribute.CollectionName;
         }
 
-        private readonly List<string> ALGORITHMS = new List<string> { Algorithms.Default };
+        private string GetUserDirectoryPath(string userId)
-        private bool IsValidAlgorithm(string algorithm)
         {
-            return ALGORITHMS.Contains(algorithm);
+            return System.IO.Path.Join(BASE_DATA_DIR, userId, GetCollectionName());
         }
 
-        public async Task<IEnumerable<T>> GetItemsSyncedAfterAsync(string userId, long timestamp)
+        private IEnumerable<string> EnumerateItems(string userId, string searchPattern = "*")
         {
-            var cursor = await Collection.FindAsync(n => (n.DateSynced > timestamp) && n.UserId.Equals(userId));
+            try
-            return cursor.ToList();
+            {
+                return System.IO.Directory.EnumerateFiles(GetUserDirectoryPath(userId), searchPattern, System.IO.SearchOption.TopDirectoryOnly);
+            }
+            catch
+            {
+                return new string[] { };
+            }
         }
 
-        // public async Task DeleteIdsAsync(string[] ids, string userId, CancellationToken token = default(CancellationToken))
+        private string FindItemById(string userId, string id)
-        // {
+        {
-        //     await Collection.DeleteManyAsync<T>((i) => ids.Contains(i.Id) && i.UserId == userId, token);
+            try
-        // }
+            {
+                var files = Directory.GetFiles(GetUserDirectoryPath(userId), $"{id}-*", System.IO.SearchOption.TopDirectoryOnly);
+                return files.Length > 0 ? files[0] : null;
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public async Task<IEnumerable<string>> GetItemsSyncedAfterAsync(string userId, long timestamp)
+        {
+            var items = new List<string>();
+            await Parallel.ForEachAsync(EnumerateItems(userId), async (file, ct) =>
+            {
+                var parts = file.Split("-");
+                var id = parts[0];
+                var dateSynced = long.Parse(parts[1]);
+                if (dateSynced > timestamp) items.Add(await File.ReadAllTextAsync(file));
+            });
+            return items;
+        }
 
         public void DeleteByUserId(string userId)
         {
-            dbContext.AddCommand((handle, ct) => Collection.DeleteManyAsync<T>(handle, (i) => i.UserId == userId, cancellationToken: ct));
+            Directory.Delete(GetUserDirectoryPath(userId), true);
         }
 
-        public async Task UpsertAsync(T item, string userId, long dateSynced)
+        public async Task UpsertAsync(string id, string item, string userId, long dateSynced)
         {
-
+            Directory.CreateDirectory(GetUserDirectoryPath(userId));
-            if (item.Length > 15 * 1024 * 1024)
+            var oldPath = FindItemById(userId, id);
-            {
+            var newPath = Path.Join(GetUserDirectoryPath(userId), $"{id}-{dateSynced}");
-                throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
+            await File.WriteAllTextAsync(newPath, item);
-            }
+            if (oldPath != null) File.Delete(oldPath);
-
-            if (!IsValidAlgorithm(item.Algorithm))
-            {
-                throw new Exception($"Invalid alg identifier {item.Algorithm}");
-            }
-
-            item.DateSynced = dateSynced;
-            item.UserId = userId;
-
-            await base.UpsertAsync(item, (x) => (x.ItemId == item.ItemId) && x.UserId == userId);
-        }
-
-        public void Upsert(T item, string userId, long dateSynced)
-        {
-
-            if (item.Length > 15 * 1024 * 1024)
-            {
-                throw new Exception($"Size of item \"{item.ItemId}\" is too large. Maximum allowed size is 15 MB.");
-            }
-
-            if (!IsValidAlgorithm(item.Algorithm))
-            {
-                throw new Exception($"Invalid alg identifier {item.Algorithm}");
-            }
-
-            item.DateSynced = dateSynced;
-            item.UserId = userId;
-
-            // await base.UpsertAsync(item, (x) => (x.ItemId == item.ItemId) && x.UserId == userId);
-            base.Upsert(item, (x) => (x.ItemId == item.ItemId) && x.UserId == userId);
         }
     }
 }