docker: use minio for s3 storage

.env

@@ -1,12 +1,6 @@
 # Required variables
 NOTESNOOK_API_SECRET= # This should be a randomly generated secret
 
-# S3 related variables for storing attachments
-S3_ACCESS_KEY=
-S3_ACCESS_KEY_ID=
-S3_SERVICE_URL=
-S3_REGION=
-
 # SMTP settings required for delivering emails
 SMTP_USERNAME=
 SMTP_PASSWORD=
@@ -31,3 +25,7 @@ SSE_SERVER_DOMAIN=
 # e.g. http://localhost:3000
 # Note: no slashes at the end
 NOTESNOOK_APP_HOST=
+
+# Minio is used for S3 storage
+MINIO_ROOT_USER= # aka. AccessKeyId (must be > 3 characters)
+MINIO_ROOT_PASSWORD= # aka. AccessKey (must be > 8 characters)

Notesnook.API/Services/S3Service.cs

@@ -34,10 +34,26 @@ using Streetwriters.Common;
 
 namespace Notesnook.API.Services
 {
+    enum S3ClientMode
+    {
+        INTERNAL = 0,
+        EXTERNAL = 1
+    }
+
     public class S3Service : IS3Service
     {
         private readonly string BUCKET_NAME = "nn-attachments";
         private AmazonS3Client S3Client { get; }
+
+        // When running in a dockerized environment the sync server doesn't have access
+        // to the host's S3 Service URL. It can only talk to S3 server via its own internal
+        // network. This creates the issue where the client needs host-level access while
+        // the sync server needs only internal access.
+        // This wouldn't be a big issue (just map one to the other right?) but the signed
+        // URLs generated by S3 are host specific. Changing their hostname on the fly causes
+        // SignatureDoesNotMatch error.
+        // That is why we create 2 separate S3 clients. One for internal traffic and one for external.
+        private AmazonS3Client S3InternalClient { get; }
         private HttpClient httpClient = new HttpClient();
 
         public S3Service()
@@ -59,6 +75,19 @@ namespace Notesnook.API.Services
 #else
             S3Client = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, config);
 #endif
+
+            if (!string.IsNullOrEmpty(Constants.S3_INTERNAL_SERVICE_URL))
+            {
+                S3InternalClient = new AmazonS3Client(Constants.S3_ACCESS_KEY_ID, Constants.S3_ACCESS_KEY, new AmazonS3Config
+                {
+                    ServiceURL = Constants.S3_INTERNAL_SERVICE_URL,
+                    AuthenticationRegion = Constants.S3_REGION,
+                    ForcePathStyle = true,
+                    SignatureMethod = SigningAlgorithm.HmacSHA256,
+                    SignatureVersion = "4"
+                });
+            }
+
             AWSConfigsS3.UseSignatureVersion4 = true;
         }
 
@@ -67,7 +96,7 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (objectName == null) throw new Exception("Invalid object name."); ;
 
-            var response = await S3Client.DeleteObjectAsync(BUCKET_NAME, objectName);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).DeleteObjectAsync(BUCKET_NAME, objectName);
 
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode)))
                 throw new Exception("Could not delete object.");
@@ -85,7 +114,7 @@ namespace Notesnook.API.Services
             var keys = new List<KeyVersion>();
             do
             {
-                response = await S3Client.ListObjectsV2Async(request);
+                response = await GetS3Client(S3ClientMode.INTERNAL).ListObjectsV2Async(request);
                 response.S3Objects.ForEach(obj => keys.Add(new KeyVersion
                 {
                     Key = obj.Key,
@@ -110,7 +139,7 @@ namespace Notesnook.API.Services
 
         public async Task<long?> GetObjectSizeAsync(string userId, string name)
         {
-            var url = this.GetPresignedURL(userId, name, HttpVerb.HEAD);
+            var url = this.GetPresignedURL(userId, name, HttpVerb.HEAD, S3ClientMode.INTERNAL);
             if (url == null) return null;
 
             var request = new HttpRequestMessage(HttpMethod.Head, url);
@@ -140,7 +169,7 @@ namespace Notesnook.API.Services
 
             if (string.IsNullOrEmpty(uploadId))
             {
-                var response = await S3Client.InitiateMultipartUploadAsync(BUCKET_NAME, objectName);
+                var response = await GetS3Client(S3ClientMode.INTERNAL).InitiateMultipartUploadAsync(BUCKET_NAME, objectName);
                 if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to initiate multipart upload.");
 
                 uploadId = response.UploadId;
@@ -164,7 +193,7 @@ namespace Notesnook.API.Services
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) throw new Exception("Could not abort multipart upload.");
 
-            var response = await S3Client.AbortMultipartUploadAsync(BUCKET_NAME, objectName, uploadId);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).AbortMultipartUploadAsync(BUCKET_NAME, objectName, uploadId);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to abort multipart upload.");
         }
 
@@ -175,11 +204,11 @@ namespace Notesnook.API.Services
 
             uploadRequest.Key = objectName;
             uploadRequest.BucketName = BUCKET_NAME;
-            var response = await S3Client.CompleteMultipartUploadAsync(uploadRequest);
+            var response = await GetS3Client(S3ClientMode.INTERNAL).CompleteMultipartUploadAsync(uploadRequest);
             if (!IsSuccessStatusCode(((int)response.HttpStatusCode))) throw new Exception("Failed to complete multipart upload.");
         }
 
-        private string GetPresignedURL(string userId, string name, HttpVerb httpVerb)
+        private string GetPresignedURL(string userId, string name, HttpVerb httpVerb, S3ClientMode mode = S3ClientMode.EXTERNAL)
         {
             var objectName = GetFullObjectName(userId, name);
             if (userId == null || objectName == null) return null;
@@ -193,15 +222,16 @@ namespace Notesnook.API.Services
 #if DEBUG
                 Protocol = Protocol.HTTP,
 #else
-                Protocol = Protocol.HTTPS,
+                Protocol = Constants.IS_SELF_HOSTED ? Protocol.HTTP : Protocol.HTTPS,
 #endif
             };
-            return S3Client.GetPreSignedURL(request);
+            return GetS3Client(mode).GetPreSignedURL(request);
         }
 
         private string GetPresignedURLForUploadPart(string objectName, string uploadId, int partNumber)
         {
-            return S3Client.GetPreSignedURL(new GetPreSignedUrlRequest
+
+            return GetS3Client().GetPreSignedURL(new GetPreSignedUrlRequest
             {
                 BucketName = BUCKET_NAME,
                 Expires = System.DateTime.Now.AddHours(1),
@@ -212,7 +242,7 @@ namespace Notesnook.API.Services
 #if DEBUG
                 Protocol = Protocol.HTTP,
 #else
-                Protocol = Protocol.HTTPS,
+                Protocol = Constants.IS_SELF_HOSTED ? Protocol.HTTP : Protocol.HTTPS,
 #endif
             });
         }
@@ -227,5 +257,11 @@ namespace Notesnook.API.Services
         {
             return ((int)statusCode >= 200) && ((int)statusCode <= 299);
         }
+
+        AmazonS3Client GetS3Client(S3ClientMode mode = S3ClientMode.EXTERNAL)
+        {
+            if (mode == S3ClientMode.INTERNAL && S3InternalClient != null) return S3InternalClient;
+            return S3Client;
+        }
     }
 }

Streetwriters.Common/Constants.cs

@@ -52,6 +52,7 @@ namespace Streetwriters.Common
         public static string ORIGIN_CERT_KEY_PATH => Environment.GetEnvironmentVariable("ORIGIN_CERT_KEY_PATH");
         public static string MONGODB_CONNECTION_STRING => Environment.GetEnvironmentVariable("MONGODB_CONNECTION_STRING");
         public static string MONGODB_DATABASE_NAME => Environment.GetEnvironmentVariable("MONGODB_DATABASE_NAME");
+        public static string S3_INTERNAL_SERVICE_URL => Environment.GetEnvironmentVariable("S3_INTERNAL_SERVICE_URL");
 
         // Server discovery
         public static int NOTESNOOK_SERVER_PORT => int.Parse(Environment.GetEnvironmentVariable("NOTESNOOK_SERVER_PORT"));

docker-compose.yml

@@ -40,8 +40,39 @@ services:
           rs.initiate();
           rs.status();
         EOF
+
+  notesnook-s3:
+    image: minio/minio
+    ports:
+      - 9000:9000
+      - 9090:9090
+    networks:
+      - notesnook
     volumes:
-      - /data/db
+      - ${HOME}/.notesnook/s3:/data/s3
+    environment:
+      MINIO_BROWSER: "on"
+    env_file:
+      - ./.env.local
+    command: server /data/s3 --console-address :9090
+
+  # There's no way to specify a default bucket in Minio so we have to
+  # set it up ourselves.
+  setup-s3:
+    image: minio/mc
+    depends_on:
+      - notesnook-s3
+    networks:
+      - notesnook
+    entrypoint: /bin/sh
+    env_file: *env-files
+    command:
+      - -c
+      - |
+        until mc config host add minio http://notesnook-s3:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD; do
+          sleep 1;
+        done;
+        mc mb minio/nn-attachments -p
 
   identity-server:
     build:
@@ -52,6 +83,8 @@ services:
     networks:
       - notesnook
     env_file: *env-files
+    depends_on:
+      - notesnook-db
     environment:
       <<: *server-discovery
       MONGODB_CONNECTION_STRING: mongodb://notesnook-db:27017/identity?replSet=rs0
@@ -66,10 +99,19 @@ services:
     networks:
       - notesnook
     env_file: *env-files
+    depends_on:
+      - notesnook-s3
+      - setup-s3
+      - identity-server
     environment:
       <<: *server-discovery
       MONGODB_CONNECTION_STRING: mongodb://notesnook-db:27017/notesnook?replSet=rs0
       MONGODB_DATABASE_NAME: notesnook
+      S3_INTERNAL_SERVICE_URL: http://notesnook-s3:9000
+      S3_ACCESS_KEY_ID: "${MINIO_ROOT_USER:-minioadmin}"
+      S3_ACCESS_KEY: "${MINIO_ROOT_PASSWORD:-minioadmin}"
+      S3_SERVICE_URL: http://localhost:9000
+      S3_REGION: us-east-1
 
   sse-server:
     build:
@@ -78,6 +120,9 @@ services:
     ports:
       - "7264:80"
     env_file: *env-files
+    depends_on:
+      - identity-server
+      - notesnook-server
     networks:
       - notesnook
     environment: