Merge commit '7621e2f8dec938cf48181c8b10afc9b01f444e68' into beta

submodules/Postbox/.gitignore

@@ -0,0 +1,25 @@
+fastlane/README.md
+fastlane/report.xml
+fastlane/test_output/*
+*.pbxuser
+!default.pbxuser
+*.mode1v3
+!default.mode1v3
+*.mode2v3
+!default.mode2v3
+*.perspectivev3
+!default.perspectivev3
+xcuserdata
+*.xccheckout
+*.xcscmblueprint
+*.moved-aside
+DerivedData
+*.hmap
+*.ipa
+*.xcuserstate
+.DS_Store
+*.dSYM
+*.dSYM.zip
+*.ipa
+*/xcuserdata/*
+Postbox.xcodeproj/*

submodules/Postbox/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "submodules/lmdb"]
+	path = submodules/lmdb
+	url = https://github.com/LMDB/lmdb.git

submodules/Postbox/BUILD

@@ -0,0 +1,26 @@
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
+
+swift_library(
+    name = "Postbox",
+    module_name = "Postbox",
+    srcs = glob([
+        "Sources/**/*.swift",
+    ]),
+    copts = [
+        "-warnings-as-errors",
+    ],
+    deps = [
+        "//submodules/Crc32:Crc32",
+    	"//submodules/SSignalKit/SwiftSignalKit:SwiftSignalKit",
+    	"//submodules/sqlcipher:sqlcipher",
+        "//submodules/MurMurHash32:MurMurHash32",
+        "//submodules/StringTransliteration:StringTransliteration",
+        "//submodules/ManagedFile:ManagedFile",
+        "//submodules/Utils/RangeSet:RangeSet",
+        "//submodules/CryptoUtils",
+        "//submodules/Utils/DarwinDirStat",
+    ],
+    visibility = [
+        "//visibility:public",
+    ],
+)

submodules/Postbox/Package.swift

@@ -0,0 +1,44 @@
+// swift-tools-version:5.5
+// The swift-tools-version declares the minimum version of Swift required to build this package.
+
+import PackageDescription
+
+let package = Package(
+    name: "Postbox",
+    platforms: [.macOS(.v10_13)],
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "Postbox",
+            targets: ["Postbox"]),
+    ],
+    dependencies: [
+        // Dependencies declare other packages that this package depends on.
+        // .package(url: /* package url */, from: "1.0.0"),
+        .package(name: "MurMurHash32", path: "../MurMurHash32"),
+        .package(name: "Crc32", path: "../Crc32"),
+        .package(name: "sqlcipher", path: "../sqlcipher"),
+        .package(name: "StringTransliteration", path: "../StringTransliteration"),
+        .package(name: "ManagedFile", path: "../ManagedFile"),
+        .package(name: "RangeSet", path: "../Utils/RangeSet"),
+        .package(name: "DarwinDirStat", path: "../Utils/DarwinDirStat"),
+        .package(name: "SSignalKit", path: "../SSignalKit"),
+        .package(name: "CryptoUtils", path: "../CryptoUtils")
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+        // Targets can depend on other targets in this package, and on products in packages this package depends on.
+        .target(
+            name: "Postbox",
+            dependencies: [.product(name: "MurMurHash32", package: "MurMurHash32", condition: nil),
+                            .product(name: "SwiftSignalKit", package: "SSignalKit", condition: nil),
+                           .product(name: "ManagedFile", package: "ManagedFile", condition: nil),
+                           .product(name: "RangeSet", package: "RangeSet", condition: nil),
+                           .product(name: "sqlcipher", package: "sqlcipher", condition: nil),
+                           .product(name: "StringTransliteration", package: "StringTransliteration", condition: nil),
+                           .product(name: "DarwinDirStat", package: "DarwinDirStat", condition: nil),
+                           .product(name: "CryptoUtils", package: "CryptoUtils", condition: nil),
+                           .product(name: "Crc32", package: "Crc32", condition: nil)],
+            path: "Sources"),
+    ]
+)

submodules/Postbox/Sources/AdditionalChatListItemsTable.swift

@@ -0,0 +1,108 @@
+import Foundation
+
+public protocol AdditionalChatListItem: PostboxCoding {
+    var peerId: PeerId { get }
+    var includeIfNoHistory: Bool { get }
+    
+    func isEqual(to other: AdditionalChatListItem) -> Bool
+}
+
+final class AdditionalChatListItemsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private var cachedItems: [AdditionalChatListItem]?
+    private var updatedItems = false
+    
+    private func key(_ index: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setInt32(0, value: index)
+        return key
+    }
+    
+    private func lowerBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1)
+        key.setInt8(0, value: 0)
+        return key
+    }
+    
+    private func upperBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setInt32(0, value: Int32.max)
+        return key
+    }
+    
+    func set(_ items: [AdditionalChatListItem]) -> Bool {
+        let current = self.get()
+        var updated = false
+        if current.count != items.count {
+            updated = true
+        } else {
+            for i in 0 ..< current.count {
+                if !current[i].isEqual(to: items[i]) {
+                    updated = true
+                    break
+                }
+            }
+        }
+        if !updated {
+            return false
+        }
+        self.cachedItems = items
+        self.updatedItems = true
+        
+        return true
+    }
+    
+    func get() -> [AdditionalChatListItem] {
+        if let cachedItems = self.cachedItems {
+            return cachedItems
+        }
+        var items: [AdditionalChatListItem] = []
+        self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), values: { key, value in
+            assert(key.getInt32(0) == Int32(items.count))
+            if value.length <= 8 {
+                return true
+            }
+            if let decoded = PostboxDecoder(buffer: value).decodeRootObject() as? AdditionalChatListItem {
+                items.append(decoded)
+            }
+            return true
+        }, limit: 0)
+        self.cachedItems = items
+        return items
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedItems = nil
+        assert(!self.updatedItems)
+    }
+    
+    override func beforeCommit() {
+        if self.updatedItems {
+            var keys: [ValueBoxKey] = []
+            self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in
+                keys.append(key)
+                return true
+            }, limit: 0)
+            for key in keys {
+                self.valueBox.remove(self.table, key: key, secure: false)
+            }
+            
+            if let items = self.cachedItems {
+                var index: Int32 = 0
+                for item in items {
+                    let encoder = PostboxEncoder()
+                    encoder.encodeRootObject(item)
+                    self.valueBox.set(self.table, key: self.key(index), value: encoder.memoryBuffer())
+                    index += 1
+                }
+            } else {
+                assertionFailure()
+            }
+            
+            self.updatedItems = false
+        }
+    }
+}

submodules/Postbox/Sources/AdditionalChatListItemsView.swift

@@ -0,0 +1,33 @@
+import Foundation
+
+final class MutableAdditionalChatListItemsView: MutablePostboxView {
+    fileprivate var items: [AdditionalChatListItem]
+    
+    init(postbox: PostboxImpl) {
+        self.items = postbox.additionalChatListItemsTable.get()
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if let items = transaction.replacedAdditionalChatListItems {
+            self.items = items
+            return true
+        }
+        return false
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return AdditionalChatListItemsView(self)
+    }
+}
+
+public final class AdditionalChatListItemsView: PostboxView {
+    public let items: [AdditionalChatListItem]
+    
+    init(_ view: MutableAdditionalChatListItemsView) {
+        self.items = view.items
+    }
+}

submodules/Postbox/Sources/AdditionalMessageHistoryViewData.swift

@@ -0,0 +1,27 @@
+import Foundation
+
+public enum AdditionalMessageHistoryViewData: Equatable {
+    case cachedPeerData(PeerId)
+    case cachedPeerDataMessages(PeerId)
+    case peerChatState(PeerId)
+    case totalUnreadState
+    case peerNotificationSettings(PeerId)
+    case cacheEntry(ItemCacheEntryId)
+    case preferencesEntry(ValueBoxKey)
+    case peer(PeerId)
+    case peerIsContact(PeerId)
+    case message(MessageId)
+}
+
+public enum AdditionalMessageHistoryViewDataEntry {
+    case cachedPeerData(PeerId, CachedPeerData?)
+    case cachedPeerDataMessages(PeerId, [MessageId: Message]?)
+    case peerChatState(PeerId, PeerChatState?)
+    case totalUnreadState(ChatListTotalUnreadState)
+    case peerNotificationSettings(PeerNotificationSettings?)
+    case cacheEntry(ItemCacheEntryId, CodableEntry?)
+    case preferencesEntry(ValueBoxKey, PreferencesEntry?)
+    case peerIsContact(PeerId, Bool)
+    case peer(PeerId, Peer?)
+    case message(MessageId, [Message])
+}

submodules/Postbox/Sources/AllChatListHolesView.swift

@@ -0,0 +1,67 @@
+import Foundation
+
+final class MutableAllChatListHolesView: MutablePostboxView {
+    fileprivate let groupId: PeerGroupId
+    private var holes = Set<ChatListHole>()
+    fileprivate var latestHole: ChatListHole?
+    
+    init(postbox: PostboxImpl, groupId: PeerGroupId) {
+        self.groupId = groupId
+        self.holes = Set(postbox.chatListTable.allHoles(groupId: groupId))
+        self.latestHole = self.holes.max(by: { $0.index < $1.index })
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if let operations = transaction.chatListOperations[self.groupId] {
+            var updated = false
+            for operation in operations {
+                switch operation {
+                case let .InsertHole(hole):
+                    if !self.holes.contains(hole) {
+                        self.holes.insert(hole)
+                        updated = true
+                    }
+                case let .RemoveHoles(indices):
+                    for index in indices {
+                        if self.holes.contains(ChatListHole(index: index.messageIndex)) {
+                            self.holes.remove(ChatListHole(index: index.messageIndex))
+                            updated = true
+                        }
+                    }
+                default:
+                    break
+                }
+            }
+            
+            if updated {
+                let updatedLatestHole = self.holes.max(by: { $0.index < $1.index })
+                if updatedLatestHole != self.latestHole {
+                    self.latestHole = updatedLatestHole
+                    return true
+                } else {
+                    return false
+                }
+            } else {
+                return false
+            }
+        } else {
+            return false
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return AllChatListHolesView(self)
+    }
+}
+
+public final class AllChatListHolesView: PostboxView {
+    public let latestHole: ChatListHole?
+    
+    init(_ view: MutableAllChatListHolesView) {
+        self.latestHole = view.latestHole
+    }
+}

submodules/Postbox/Sources/BinarySearch.swift

@@ -0,0 +1,100 @@
+import Foundation
+
+func binarySearch<A, T: Comparable>(_ inputArr: [A], extract: (A) -> T, searchItem: T) -> Int? {
+    var lowerIndex = 0
+    var upperIndex = inputArr.count - 1
+    
+    if lowerIndex > upperIndex {
+        return nil
+    }
+    
+    while true {
+        let currentIndex = (lowerIndex + upperIndex) / 2
+        let value = extract(inputArr[currentIndex])
+        
+        if value == searchItem {
+            return currentIndex
+        } else if lowerIndex > upperIndex {
+            return nil
+        } else {
+            if (value > searchItem) {
+                upperIndex = currentIndex - 1
+            } else {
+                lowerIndex = currentIndex + 1
+            }
+        }
+    }
+}
+
+func binarySearch<T: Comparable>(_ inputArr: [T], searchItem: T) -> Int? {
+    var lowerIndex = 0;
+    var upperIndex = inputArr.count - 1
+    
+    if lowerIndex > upperIndex {
+        return nil
+    }
+    
+    while (true) {
+        let currentIndex = (lowerIndex + upperIndex) / 2
+        if (inputArr[currentIndex] == searchItem) {
+            return currentIndex
+        } else if (lowerIndex > upperIndex) {
+            return nil
+        } else {
+            if (inputArr[currentIndex] > searchItem) {
+                upperIndex = currentIndex - 1
+            } else {
+                lowerIndex = currentIndex + 1
+            }
+        }
+    }
+}
+
+func binaryInsertionIndex<A, T: Comparable>(_ inputArr: [A], extract: (A) -> T, searchItem: T) -> Int {
+    var lo = 0
+    var hi = inputArr.count - 1
+    while lo <= hi {
+        let mid = (lo + hi) / 2
+        let value = extract(inputArr[mid])
+        if value < searchItem {
+            lo = mid + 1
+        } else if searchItem < value {
+            hi = mid - 1
+        } else {
+            return mid
+        }
+    }
+    return lo
+}
+
+func binaryInsertionIndex<T: Comparable>(_ inputArr: [T], searchItem: T) -> Int {
+    var lo = 0
+    var hi = inputArr.count - 1
+    while lo <= hi {
+        let mid = (lo + hi) / 2
+        if inputArr[mid] < searchItem {
+            lo = mid + 1
+        } else if searchItem < inputArr[mid] {
+            hi = mid - 1
+        } else {
+            return mid
+        }
+    }
+    return lo
+}
+
+func binaryInsertionIndexReverse<T: Comparable>(_ inputArr: [T], searchItem: T) -> Int {
+    var lo = 0
+    var hi = inputArr.count - 1
+    while lo <= hi {
+        let mid = (lo + hi) / 2
+        if inputArr[mid] > searchItem {
+            lo = mid + 1
+        } else if searchItem > inputArr[mid] {
+            hi = mid - 1
+        } else {
+            return mid
+        }
+    }
+    return lo
+}

submodules/Postbox/Sources/CachedItemView.swift

@@ -0,0 +1,35 @@
+import Foundation
+
+final class MutableCachedItemView: MutablePostboxView {
+    private let id: ItemCacheEntryId
+    fileprivate var value: CodableEntry?
+    
+    init(postbox: PostboxImpl, id: ItemCacheEntryId) {
+        self.id = id
+        self.value = postbox.itemCacheTable.retrieve(id: id, metaTable: postbox.itemCacheMetaTable)
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if transaction.updatedCacheEntryKeys.contains(self.id) {
+            self.value = postbox.itemCacheTable.retrieve(id: id, metaTable: postbox.itemCacheMetaTable)
+            return true
+        }
+        return false
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return CachedItemView(self)
+    }
+}
+
+public final class CachedItemView: PostboxView {
+    public let value: CodableEntry?
+    
+    init(_ view: MutableCachedItemView) {
+        self.value = view.value
+    }
+}

submodules/Postbox/Sources/CachedPeerData.swift

@@ -0,0 +1,9 @@
+
+public protocol CachedPeerData: AnyObject, PostboxCoding {
+    var peerIds: Set<PeerId> { get }
+    var messageIds: Set<MessageId> { get }
+    
+    var associatedHistoryMessageId: MessageId? { get }
+    
+    func isEqual(to: CachedPeerData) -> Bool
+}

submodules/Postbox/Sources/CachedPeerDataTable.swift

@@ -0,0 +1,72 @@
+import Foundation
+
+final class CachedPeerDataTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: false)
+    }
+    
+    private let sharedEncoder = PostboxEncoder()
+    private let sharedKey = ValueBoxKey(length: 8)
+    
+    private var cachedDatas: [PeerId: CachedPeerData] = [:]
+    private var updatedPeers: [PeerId: (CachedPeerData?, CachedPeerData)] = [:]
+    
+    override init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool) {
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(_ id: PeerId) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: id.toInt64())
+        return self.sharedKey
+    }
+    
+    func set(id: PeerId, data: CachedPeerData) {
+        var previousValue: CachedPeerData?
+        if let currentUpdate = self.updatedPeers[id] {
+            previousValue = currentUpdate.0
+        } else {
+            previousValue = self.get(id)
+        }
+        
+        self.cachedDatas[id] = data
+        self.updatedPeers[id] = (previousValue, data)
+    }
+    
+    func get(_ id: PeerId) -> CachedPeerData? {
+        if let status = self.cachedDatas[id] {
+            return status
+        }
+        if let value = self.valueBox.get(self.table, key: self.key(id)) {
+            if let data = PostboxDecoder(buffer: value).decodeRootObject() as? CachedPeerData {
+                self.cachedDatas[id] = data
+                return data
+            }
+        }
+        return nil
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedDatas.removeAll()
+        self.updatedPeers.removeAll()
+    }
+    
+    func transactionUpdatedPeers() -> [PeerId: (CachedPeerData?, CachedPeerData)] {
+        return self.updatedPeers
+    }
+    
+    override func beforeCommit() {
+        for peerId in self.updatedPeers.keys {
+            if let data = self.cachedDatas[peerId] {
+                self.sharedEncoder.reset()
+                self.sharedEncoder.encodeRootObject(data)
+                
+                self.valueBox.set(self.table, key: self.key(peerId), value: self.sharedEncoder.readBufferNoCopy())
+            }
+        }
+        
+        self.updatedPeers.removeAll()
+        if !self.useCaches {
+            self.cachedDatas.removeAll()
+        }
+    }
+}

submodules/Postbox/Sources/CachedPeerDataView.swift

@@ -0,0 +1,95 @@
+import Foundation
+
+final class MutableCachedPeerDataView: MutablePostboxView {
+    let peerId: PeerId
+    let trackAssociatedMessages: Bool
+    var cachedPeerData: CachedPeerData?
+    var associatedMessages: [MessageId: Message] = [:]
+    
+    init(postbox: PostboxImpl, peerId: PeerId, trackAssociatedMessages: Bool) {
+        self.peerId = peerId
+        self.trackAssociatedMessages = trackAssociatedMessages
+        self.cachedPeerData = postbox.cachedPeerDataTable.get(peerId)
+        if let cachedPeerData = self.cachedPeerData {
+            for id in cachedPeerData.messageIds {
+                if let message = postbox.getMessage(id) {
+                    self.associatedMessages[message.id] = message
+                }
+            }
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if let cachedPeerData = transaction.currentUpdatedCachedPeerData[self.peerId]?.updated {
+            self.cachedPeerData = cachedPeerData
+            if self.trackAssociatedMessages {
+                self.associatedMessages.removeAll()
+                for id in cachedPeerData.messageIds {
+                    if let message = postbox.getMessage(id) {
+                        self.associatedMessages[message.id] = message
+                    }
+                }
+            }
+            
+            return true
+        } else {
+            var updatedIds = Set<MessageId>()
+            if self.trackAssociatedMessages {
+                if let cachedPeerData = self.cachedPeerData {
+                    for peerId in Set(cachedPeerData.messageIds.map(\.peerId)) {
+                        if let operations = transaction.currentOperationsByPeerId[peerId] {
+                            for operation in operations {
+                                switch operation {
+                                case let .InsertMessage(message):
+                                    if cachedPeerData.messageIds.contains(message.id) {
+                                        updatedIds.insert(message.id)
+                                    }
+                                case let .Remove(indices):
+                                    for index in indices {
+                                        if cachedPeerData.messageIds.contains(index.0.id) {
+                                            updatedIds.insert(index.0.id)
+                                        }
+                                    }
+                                default:
+                                    break
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if !updatedIds.isEmpty {
+                for id in updatedIds {
+                    if let message = postbox.getMessage(id) {
+                        self.associatedMessages[message.id] = message
+                    } else {
+                        self.associatedMessages.removeValue(forKey: id)
+                    }
+                }
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return CachedPeerDataView(self)
+    }
+}
+
+public final class CachedPeerDataView: PostboxView {
+    public let peerId: PeerId
+    public let cachedPeerData: CachedPeerData?
+    public let associatedMessages: [MessageId: Message]
+    
+    init(_ view: MutableCachedPeerDataView) {
+        self.peerId = view.peerId
+        self.cachedPeerData = view.cachedPeerData
+        self.associatedMessages = view.associatedMessages
+    }
+}

submodules/Postbox/Sources/ChatInterfaceStateView.swift

@@ -0,0 +1,48 @@
+import Foundation
+
+final class MutableChatInterfaceStateView: MutablePostboxView {
+    fileprivate let peerId: PeerId
+    fileprivate var value: StoredPeerChatInterfaceState?
+    
+    init(postbox: PostboxImpl, peerId: PeerId) {
+        self.peerId = peerId
+        
+        self.reload(postbox: postbox)
+    }
+    
+    private func reload(postbox: PostboxImpl) {
+        self.value = postbox.peerChatInterfaceStateTable.get(self.peerId)
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if transaction.currentUpdatedPeerChatListEmbeddedStates.contains(self.peerId) {
+            let previousValue = self.value
+            self.reload(postbox: postbox)
+            if previousValue != self.value {
+                updated = true
+            }
+        }
+        
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        self.reload(postbox: postbox)
+        
+        return true
+    }
+    
+    func immutableView() -> PostboxView {
+        return ChatInterfaceStateView(self)
+    }
+}
+
+public final class ChatInterfaceStateView: PostboxView {
+    public let value: StoredPeerChatInterfaceState?
+    
+    init(_ view: MutableChatInterfaceStateView) {
+        self.value = view.value
+    }
+}

submodules/Postbox/Sources/ChatListHole.swift

@@ -0,0 +1,17 @@
+import Foundation
+
+public struct ChatListHole: Hashable, CustomStringConvertible {
+    public let index: MessageIndex
+    
+    public init(index: MessageIndex) {
+        self.index = index
+    }
+    
+    public var description: String {
+        return "ChatListHole(\(self.index.id), \(self.index.timestamp))"
+    }
+
+    public static func <(lhs: ChatListHole, rhs: ChatListHole) -> Bool {
+        return lhs.index < rhs.index
+    }
+}

submodules/Postbox/Sources/ChatListHolesView.swift

@@ -0,0 +1,64 @@
+import Foundation
+
+public struct ChatListHolesEntry: Hashable {
+    public let groupId: PeerGroupId
+    public let hole: ChatListHole
+    
+    public init(groupId: PeerGroupId, hole: ChatListHole) {
+        self.groupId = groupId
+        self.hole = hole
+    }
+}
+
+final class MutableChatListHolesView {
+    fileprivate var entries = Set<ChatListHolesEntry>()
+    
+    func update(holes: Set<ChatListHolesEntry>) -> Bool {
+        if self.entries != holes {
+            self.entries = holes
+            return true
+        } else {
+            return false
+        }
+    }
+}
+
+public final class ChatListHolesView {
+    public let entries: Set<ChatListHolesEntry>
+    
+    init(_ mutableView: MutableChatListHolesView) {
+        self.entries = mutableView.entries
+    }
+}
+
+public struct ForumTopicListHolesEntry: Hashable {
+    public let peerId: PeerId
+    public let index: StoredPeerThreadCombinedState.Index?
+    
+    public init(peerId: PeerId, index: StoredPeerThreadCombinedState.Index?) {
+        self.peerId = peerId
+        self.index = index
+    }
+}
+
+final class MutableForumTopicListHolesView {
+    fileprivate var entries = Set<ForumTopicListHolesEntry>()
+    
+    func update(holes: Set<ForumTopicListHolesEntry>) -> Bool {
+        if self.entries != holes {
+            self.entries = holes
+            return true
+        } else {
+            return false
+        }
+    }
+}
+
+public final class ForumTopicListHolesView {
+    public let entries: Set<ForumTopicListHolesEntry>
+    
+    init(_ mutableView: MutableForumTopicListHolesView) {
+        self.entries = mutableView.entries
+    }
+}
+

submodules/Postbox/Sources/ChatListIndexTable.swift

@@ -0,0 +1,843 @@
+import Foundation
+
+struct ChatListPeerInclusionIndex {
+    let topMessageIndex: MessageIndex?
+    let inclusion: PeerChatListInclusion
+    
+    func includedIndex(peerId: PeerId) -> (PeerGroupId, ChatListIndex)? {
+        switch inclusion {
+            case .notIncluded:
+                return nil
+            case let .ifHasMessagesOrOneOf(groupId, pinningIndex, minTimestamp):
+                if let minTimestamp = minTimestamp {
+                    if let topMessageIndex = self.topMessageIndex, topMessageIndex.timestamp >= minTimestamp {
+                        return (groupId, ChatListIndex(pinningIndex: pinningIndex, messageIndex: topMessageIndex))
+                    } else {
+                        return (groupId, ChatListIndex(pinningIndex: pinningIndex, messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: minTimestamp)))
+                    }
+                } else if let topMessageIndex = self.topMessageIndex {
+                    return (groupId, ChatListIndex(pinningIndex: pinningIndex, messageIndex: topMessageIndex))
+                } else if let pinningIndex = pinningIndex {
+                    return (groupId, ChatListIndex(pinningIndex: pinningIndex, messageIndex: MessageIndex(id: MessageId(peerId: peerId, namespace: 0, id: 0), timestamp: 0)))
+                } else {
+                    return nil
+                }
+        }
+    }
+}
+
+private struct ChatListIndexFlags: OptionSet {
+    var rawValue: Int8
+    
+    init(rawValue: Int8) {
+        self.rawValue = rawValue
+    }
+    
+    static let hasIndex = ChatListIndexFlags(rawValue: 1 << 0)
+}
+
+final class ChatListIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: true)
+    }
+    
+    private let peerNameIndexTable: PeerNameIndexTable
+    private let metadataTable: MessageHistoryMetadataTable
+    private let readStateTable: MessageHistoryReadStateTable
+    private let notificationSettingsTable: PeerNotificationSettingsTable
+    
+    private let sharedKey = ValueBoxKey(length: 8)
+    
+    private var cachedPeerIndices: [PeerId: ChatListPeerInclusionIndex] = [:]
+    
+    private var updatedPreviousPeerCachedIndices: [PeerId: ChatListPeerInclusionIndex] = [:]
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, peerNameIndexTable: PeerNameIndexTable, metadataTable: MessageHistoryMetadataTable, readStateTable: MessageHistoryReadStateTable, notificationSettingsTable: PeerNotificationSettingsTable) {
+        self.peerNameIndexTable = peerNameIndexTable
+        self.metadataTable = metadataTable
+        self.readStateTable = readStateTable
+        self.notificationSettingsTable = notificationSettingsTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(_ peerId: PeerId) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: peerId.toInt64())
+        assert(self.sharedKey.getInt64(0) == peerId.toInt64())
+        return self.sharedKey
+    }
+    
+    private func key(_ groupId: PeerGroupId) -> ValueBoxKey {
+        self.sharedKey.setInt32(0, value: Int32.max)
+        self.sharedKey.setInt32(4, value: groupId.rawValue)
+        return self.sharedKey
+    }
+    
+    func setTopMessageIndex(peerId: PeerId, index: MessageIndex?) -> ChatListPeerInclusionIndex {
+        let current = self.get(peerId: peerId)
+        if self.updatedPreviousPeerCachedIndices[peerId] == nil {
+            self.updatedPreviousPeerCachedIndices[peerId] = current
+        }
+        let updated = ChatListPeerInclusionIndex(topMessageIndex: index, inclusion: current.inclusion)
+        self.cachedPeerIndices[peerId] = updated
+        return updated
+    }
+    
+    func setInclusion(peerId: PeerId, inclusion: PeerChatListInclusion) -> ChatListPeerInclusionIndex {
+        let current = self.get(peerId: peerId)
+        if self.updatedPreviousPeerCachedIndices[peerId] == nil {
+            self.updatedPreviousPeerCachedIndices[peerId] = current
+        }
+        let updated = ChatListPeerInclusionIndex(topMessageIndex: current.topMessageIndex, inclusion: inclusion)
+        self.cachedPeerIndices[peerId] = updated
+        return updated
+    }
+    
+    func get(peerId: PeerId) -> ChatListPeerInclusionIndex {
+        if let cached = self.cachedPeerIndices[peerId] {
+            return cached
+        } else {
+            if let value = self.valueBox.get(self.table, key: self.key(peerId)) {
+                let topMessageIndex: MessageIndex?
+                
+                var flagsValue: Int8 = 0
+                value.read(&flagsValue, offset: 0, length: 1)
+                let flags = ChatListIndexFlags(rawValue: flagsValue)
+                
+                if flags.contains(.hasIndex) {
+                    var idNamespace: Int32 = 0
+                    var idId: Int32 = 0
+                    var idTimestamp: Int32 = 0
+                    value.read(&idNamespace, offset: 0, length: 4)
+                    value.read(&idId, offset: 0, length: 4)
+                    value.read(&idTimestamp, offset: 0, length: 4)
+                    topMessageIndex = MessageIndex(id: MessageId(peerId: peerId, namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                } else {
+                    topMessageIndex = nil
+                }
+                
+                let inclusion: PeerChatListInclusion
+                
+                var inclusionId: Int8 = 0
+                value.read(&inclusionId, offset: 0, length: 1)
+                if inclusionId == 0 {
+                    inclusion = .notIncluded
+                } else if inclusionId == 1 {
+                    var pinningIndexValue: UInt16 = 0
+                    value.read(&pinningIndexValue, offset: 0, length: 2)
+                    
+                    var hasMinTimestamp: Int8 = 0
+                    value.read(&hasMinTimestamp, offset: 0, length: 1)
+                    let minTimestamp: Int32?
+                    if hasMinTimestamp != 0 {
+                        var minTimestampValue: Int32 = 0
+                        value.read(&minTimestampValue, offset: 0, length: 4)
+                        minTimestamp = minTimestampValue
+                    } else {
+                        minTimestamp = nil
+                    }
+                    
+                    var groupIdValue: Int32 = 0
+                    value.read(&groupIdValue, offset: 0, length: 4)
+                    
+                    inclusion = .ifHasMessagesOrOneOf(groupId: PeerGroupId(rawValue: groupIdValue), pinningIndex: chatListPinningIndexFromKeyValue(pinningIndexValue), minTimestamp: minTimestamp)
+                } else {
+                    preconditionFailure()
+                }
+                
+                let inclusionIndex = ChatListPeerInclusionIndex(topMessageIndex: topMessageIndex, inclusion: inclusion)
+                self.cachedPeerIndices[peerId] = inclusionIndex
+                return inclusionIndex
+            } else {
+                let inclusionIndex = ChatListPeerInclusionIndex(topMessageIndex: nil, inclusion: .notIncluded)
+                self.cachedPeerIndices[peerId] = inclusionIndex
+                return inclusionIndex
+            }
+        }
+    }
+    
+    func getAllPeerIds() -> [PeerId] {
+        var peerIds: [PeerId] = []
+        self.valueBox.scanInt64(self.table, keys: { key in
+            peerIds.append(PeerId(key))
+            return true
+        })
+        return peerIds
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedPeerIndices.removeAll()
+        assert(self.updatedPreviousPeerCachedIndices.isEmpty)
+    }
+    
+    func commitWithTransaction(
+        postbox: PostboxImpl,
+        currentTransaction: Transaction,
+        alteredInitialPeerCombinedReadStates: [PeerId: CombinedPeerReadState],
+        updatedPeers: [((Peer, Bool)?, (Peer, Bool))],
+        updatedCachedPeerData: [PeerId: (CachedPeerData?, CachedPeerData)],
+        transactionParticipationInTotalUnreadCountUpdates: (added: Set<PeerId>, removed: Set<PeerId>),
+        alteredInitialPeerThreadsSummaries: [PeerId: StoredPeerThreadsSummary],
+        updatedTotalUnreadStates: inout [PeerGroupId: ChatListTotalUnreadState],
+        updatedGroupTotalUnreadSummaries: inout [PeerGroupId: PeerGroupUnreadCountersCombinedSummary],
+        currentUpdatedGroupSummarySynchronizeOperations: inout [PeerGroupAndNamespace: Bool]
+    ) {
+        var updatedPeerTags: [PeerId: (previous: PeerSummaryCounterTags, updated: PeerSummaryCounterTags)] = [:]
+        var updatedIsThreadBasedUnreadCountCalculation: [PeerId: Bool] = [:]
+        
+        var upatedPeerMap: [PeerId: (Peer?, Peer)] = [:]
+        
+        for (previous, updated) in updatedPeers {
+            var needsAssociatedPeers = false
+            if let previous, previous.0.associatedPeerId != nil {
+                needsAssociatedPeers = true
+            } else if updated.0.associatedPeerId != nil {
+                needsAssociatedPeers = true
+            }
+            
+            if needsAssociatedPeers && upatedPeerMap.isEmpty {
+                for (previous, updated) in updatedPeers {
+                    upatedPeerMap[updated.0.id] = (previous?.0, updated.0)
+                }
+            }
+            
+            var previousAssociatedPeer: Peer?
+            var updatedAssociatedPeer: Peer?
+            if let previous, let previousAssociatedPeerId = previous.0.associatedPeerId {
+                previousAssociatedPeer = upatedPeerMap[previousAssociatedPeerId]?.0 ?? postbox.peerTable.get(previousAssociatedPeerId)
+            }
+            if let updatedAssociatedPeerId = updated.0.associatedPeerId {
+                updatedAssociatedPeer = upatedPeerMap[updatedAssociatedPeerId]?.1 ?? postbox.peerTable.get(updatedAssociatedPeerId)
+            }
+            
+            let previousTags: PeerSummaryCounterTags
+            if let (previous, previousIsContact) = previous {
+                previousTags = postbox.seedConfiguration.peerSummaryCounterTags(previous, previousIsContact)
+            } else {
+                previousTags = []
+            }
+            let updatedTags = postbox.seedConfiguration.peerSummaryCounterTags(updated.0, updated.1)
+            if previousTags != updatedTags {
+                updatedPeerTags[updated.0.id] = (previousTags, updatedTags)
+            }
+            
+            if let previous = previous {
+                var isThreadBasedUnreadCalculation = postbox.seedConfiguration.peerSummaryIsThreadBased(updated.0, updatedAssociatedPeer).value
+                if let cachedData = updatedCachedPeerData[updated.0.id]?.1, postbox.seedConfiguration.decodeDisplayPeerAsRegularChat(cachedData) {
+                    isThreadBasedUnreadCalculation = false
+                }
+                
+                var wasThreadBasedUnreadCalculation = false
+                if postbox.seedConfiguration.peerSummaryIsThreadBased(previous.0, previousAssociatedPeer).value {
+                    if let cachedData = postbox.cachedPeerDataTable.get(previous.0.id), postbox.seedConfiguration.decodeDisplayPeerAsRegularChat(cachedData) {
+                    } else {
+                        wasThreadBasedUnreadCalculation = true
+                    }
+                }
+                
+                if wasThreadBasedUnreadCalculation != isThreadBasedUnreadCalculation {
+                    updatedIsThreadBasedUnreadCountCalculation[updated.0.id] = isThreadBasedUnreadCalculation
+                }
+            }
+        }
+        for (peerId, cachedDataUpdate) in updatedCachedPeerData {
+            if updatedIsThreadBasedUnreadCountCalculation[peerId] != nil {
+                continue
+            }
+            guard let peer = postbox.peerTable.get(peerId) else {
+                continue
+            }
+            let associatedPeer = peer.associatedPeerId.flatMap(postbox.peerTable.get)
+            
+            var isThreadBasedUnreadCalculation = postbox.seedConfiguration.peerSummaryIsThreadBased(peer, associatedPeer).value
+            if postbox.seedConfiguration.decodeDisplayPeerAsRegularChat(cachedDataUpdate.1) {
+                isThreadBasedUnreadCalculation = false
+            }
+            
+            var wasThreadBasedUnreadCalculation = false
+            if postbox.seedConfiguration.peerSummaryIsThreadBased(peer, associatedPeer).value {
+                if let previousCachedData = cachedDataUpdate.0, postbox.seedConfiguration.decodeDisplayPeerAsRegularChat(previousCachedData) {
+                } else {
+                    wasThreadBasedUnreadCalculation = true
+                }
+            }
+            
+            if wasThreadBasedUnreadCalculation != isThreadBasedUnreadCalculation {
+                updatedIsThreadBasedUnreadCountCalculation[peerId] = isThreadBasedUnreadCalculation
+            }
+        }
+        
+        if !self.updatedPreviousPeerCachedIndices.isEmpty || !alteredInitialPeerCombinedReadStates.isEmpty || !updatedPeerTags.isEmpty || !transactionParticipationInTotalUnreadCountUpdates.added.isEmpty || !transactionParticipationInTotalUnreadCountUpdates.removed.isEmpty || !alteredInitialPeerThreadsSummaries.isEmpty || !updatedCachedPeerData.isEmpty {
+            var addedToGroupPeerIds: [PeerId: PeerGroupId] = [:]
+            var removedFromGroupPeerIds: [PeerId: PeerGroupId] = [:]
+            var addedToIndexPeerIds = Set<PeerId>()
+            var removedFromIndexPeerIds = Set<PeerId>()
+            
+            for (peerId, previousIndex) in self.updatedPreviousPeerCachedIndices {
+                let index = self.cachedPeerIndices[peerId]!
+                if let (currentGroupId, _) = index.includedIndex(peerId: peerId) {
+                    let previousGroupId = previousIndex.includedIndex(peerId: peerId)?.0
+                    if previousGroupId != currentGroupId {
+                        addedToGroupPeerIds[peerId] = currentGroupId
+                        if let previousGroupId = previousGroupId {
+                            removedFromGroupPeerIds[peerId] = previousGroupId
+                        } else {
+                            addedToIndexPeerIds.insert(peerId)
+                        }
+                    }
+                } else if let (previousGroupId, _) = previousIndex.includedIndex(peerId: peerId) {
+                    removedFromGroupPeerIds[peerId] = previousGroupId
+                    removedFromIndexPeerIds.insert(peerId)
+                }
+                
+                let writeBuffer = WriteBuffer()
+                
+                var flags: ChatListIndexFlags = []
+                
+                if index.topMessageIndex != nil {
+                    flags.insert(.hasIndex)
+                }
+                
+                var flagsValue = flags.rawValue
+                writeBuffer.write(&flagsValue, offset: 0, length: 1)
+                
+                if let topMessageIndex = index.topMessageIndex {
+                    var idNamespace: Int32 = topMessageIndex.id.namespace
+                    var idId: Int32 = topMessageIndex.id.id
+                    var idTimestamp: Int32 = topMessageIndex.timestamp
+                    writeBuffer.write(&idNamespace, offset: 0, length: 4)
+                    writeBuffer.write(&idId, offset: 0, length: 4)
+                    writeBuffer.write(&idTimestamp, offset: 0, length: 4)
+                }
+                
+                switch index.inclusion {
+                    case .notIncluded:
+                        var key: Int8 = 0
+                        writeBuffer.write(&key, offset: 0, length: 1)
+                    case let .ifHasMessagesOrOneOf(groupId, pinningIndex, minTimestamp):
+                        var key: Int8 = 1
+                        writeBuffer.write(&key, offset: 0, length: 1)
+                    
+                        var pinningIndexValue: UInt16 = keyValueForChatListPinningIndex(pinningIndex)
+                        writeBuffer.write(&pinningIndexValue, offset: 0, length: 2)
+                    
+                        if let minTimestamp = minTimestamp {
+                            var hasMinTimestamp: Int8 = 1
+                            writeBuffer.write(&hasMinTimestamp, offset: 0, length: 1)
+                            
+                            var minTimestampValue = minTimestamp
+                            writeBuffer.write(&minTimestampValue, offset: 0, length: 4)
+                        } else {
+                            var hasMinTimestamp: Int8 = 0
+                            writeBuffer.write(&hasMinTimestamp, offset: 0, length: 1)
+                        }
+                    
+                        var groupIdValue = groupId.rawValue
+                        writeBuffer.write(&groupIdValue, offset: 0, length: 4)
+                }
+                
+                withExtendedLifetime(writeBuffer, {
+                    self.valueBox.set(self.table, key: self.key(peerId), value: writeBuffer.readBufferNoCopy())
+                })
+            }
+            self.updatedPreviousPeerCachedIndices.removeAll()
+            
+            for peerId in addedToIndexPeerIds {
+                self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.chats], includes: true)
+            }
+            
+            for peerId in removedFromIndexPeerIds {
+                self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.chats], includes: false)
+            }
+            
+            var alteredPeerIds = Set<PeerId>()
+            for (peerId, _) in alteredInitialPeerCombinedReadStates {
+                alteredPeerIds.insert(peerId)
+            }
+            for (peerId, _) in alteredInitialPeerThreadsSummaries {
+                alteredPeerIds.insert(peerId)
+            }
+            
+            alteredPeerIds.formUnion(addedToGroupPeerIds.keys)
+            alteredPeerIds.formUnion(removedFromGroupPeerIds.keys)
+            alteredPeerIds.formUnion(transactionParticipationInTotalUnreadCountUpdates.added)
+            alteredPeerIds.formUnion(transactionParticipationInTotalUnreadCountUpdates.removed)
+            alteredPeerIds.formUnion(updatedCachedPeerData.keys)
+            
+            for peerId in updatedPeerTags.keys {
+                alteredPeerIds.insert(peerId)
+            }
+            
+            for peerId in updatedIsThreadBasedUnreadCountCalculation.keys {
+                alteredPeerIds.insert(peerId)
+            }
+            
+            var additionalAlteredPeerIds = Set<PeerId>()
+            for peerId in alteredPeerIds {
+                guard let peer = postbox.peerTable.get(peerId) else {
+                    continue
+                }
+                if let associatedPeerId = peer.associatedPeerId {
+                    additionalAlteredPeerIds.insert(associatedPeerId)
+                }
+                if let reverseAssociatedPeerId = postbox.reverseAssociatedPeerTable.get(peerId: peerId).first {
+                    additionalAlteredPeerIds.insert(reverseAssociatedPeerId)
+                }
+            }
+            alteredPeerIds.formUnion(additionalAlteredPeerIds)
+            
+            func alterTags(_ totalUnreadState: inout ChatListTotalUnreadState, _ peerId: PeerId, _ tag: PeerSummaryCounterTags, _ f: (ChatListTotalUnreadCounters, ChatListTotalUnreadCounters) -> (ChatListTotalUnreadCounters, ChatListTotalUnreadCounters)) {
+                if totalUnreadState.absoluteCounters[tag] == nil {
+                    totalUnreadState.absoluteCounters[tag] = ChatListTotalUnreadCounters(messageCount: 0, chatCount: 0)
+                }
+                if totalUnreadState.filteredCounters[tag] == nil {
+                    totalUnreadState.filteredCounters[tag] = ChatListTotalUnreadCounters(messageCount: 0, chatCount: 0)
+                }
+                var (updatedAbsoluteCounters, updatedFilteredCounters) = f(totalUnreadState.absoluteCounters[tag]!, totalUnreadState.filteredCounters[tag]!)
+                if updatedAbsoluteCounters.messageCount < 0 {
+                    updatedAbsoluteCounters.messageCount = 0
+                }
+                if updatedAbsoluteCounters.chatCount < 0 {
+                    updatedAbsoluteCounters.chatCount = 0
+                }
+                if updatedFilteredCounters.messageCount < 0 {
+                    updatedFilteredCounters.messageCount = 0
+                }
+                if updatedFilteredCounters.chatCount < 0 {
+                    updatedFilteredCounters.chatCount = 0
+                }
+                totalUnreadState.absoluteCounters[tag] = updatedAbsoluteCounters
+                totalUnreadState.filteredCounters[tag] = updatedFilteredCounters
+            }
+            
+            func alterNamespace(summary: inout PeerGroupUnreadCountersSummary, previousState: PeerReadState?, updatedState: PeerReadState?) {
+                let previousCount = previousState?.count ?? 0
+                let updatedCount = updatedState?.count ?? 0
+                if previousCount != updatedCount {
+                    if (previousCount != 0) != (updatedCount != 0) {
+                        if updatedCount != 0 {
+                            summary.all.chatCount += 1
+                        } else {
+                            summary.all.chatCount -= 1
+                            summary.all.chatCount = max(0, summary.all.chatCount)
+                        }
+                    }
+                    summary.all.messageCount += updatedCount - previousCount
+                    summary.all.messageCount = max(0, summary.all.messageCount)
+                }
+            }
+            
+            var updatedTotalStates: [PeerGroupId: ChatListTotalUnreadState] = [:]
+            var updatedTotalUnreadSummaries: [PeerGroupId: PeerGroupUnreadCountersCombinedSummary] = [:]
+            
+            let globalNotificationSettings = postbox.getGlobalNotificationSettings(transaction: currentTransaction)
+            
+            for peerId in alteredPeerIds {
+                guard let peer = postbox.peerTable.get(peerId) else {
+                    continue
+                }
+                let isContact = postbox.contactsTable.isContact(peerId: peerId)
+                let notificationPeerId: PeerId
+                if let associatedPeerId = peer.associatedPeerId, peer.associatedPeerOverridesIdentity {
+                    notificationPeerId = associatedPeerId
+                } else {
+                    notificationPeerId = peerId
+                }
+                
+                let initialReadState: CombinedPeerReadState?
+                if let updated = updatedIsThreadBasedUnreadCountCalculation[peerId] {
+                    if updated {
+                        // was not thread-based, use peer read state
+                        initialReadState = alteredInitialPeerCombinedReadStates[peerId] ?? postbox.readStateTable.getCombinedState(peerId)
+                    } else {
+                        let previousCount: Int32
+                        if let previousSummary = alteredInitialPeerThreadsSummaries[peerId] {
+                            previousCount = previousSummary.effectiveUnreadCount
+                        } else {
+                            previousCount = postbox.peerThreadsSummaryTable.get(peerId: peerId)?.effectiveUnreadCount ?? 0
+                        }
+                        initialReadState = CombinedPeerReadState(states: [(0, .idBased(maxIncomingReadId: 0, maxOutgoingReadId: 1, maxKnownId: 0, count: previousCount, markedUnread: false))])
+                    }
+                } else {
+                    var displayAsRegularChat = false
+                    if let cachedData = postbox.cachedPeerDataTable.get(peerId), postbox.seedConfiguration.decodeDisplayPeerAsRegularChat(cachedData) {
+                        displayAsRegularChat = true
+                    }
+                    
+                    if let peer = postbox.peerTable.get(peerId), postbox.seedConfiguration.peerSummaryIsThreadBased(peer, peer.associatedPeerId.flatMap(postbox.peerTable.get)).value, !displayAsRegularChat {
+                        let previousCount: Int32
+                        if let previousSummary = alteredInitialPeerThreadsSummaries[peerId] {
+                            previousCount = previousSummary.effectiveUnreadCount
+                        } else {
+                            previousCount = postbox.peerThreadsSummaryTable.get(peerId: peerId)?.effectiveUnreadCount ?? 0
+                        }
+                        initialReadState = CombinedPeerReadState(states: [(0, .idBased(maxIncomingReadId: 0, maxOutgoingReadId: 1, maxKnownId: 0, count: previousCount, markedUnread: false))])
+                    } else {
+                        initialReadState = alteredInitialPeerCombinedReadStates[peerId] ?? postbox.readStateTable.getCombinedState(peerId)
+                    }
+                }
+                
+                var displayAsRegularChat = false
+                if let cachedData = postbox.cachedPeerDataTable.get(peerId), postbox.seedConfiguration.decodeDisplayPeerAsRegularChat(cachedData) {
+                    displayAsRegularChat = true
+                }
+                
+                let currentReadState: CombinedPeerReadState?
+                if let peer = postbox.peerTable.get(peerId), postbox.seedConfiguration.peerSummaryIsThreadBased(peer, peer.associatedPeerId.flatMap(postbox.peerTable.get)).value, !displayAsRegularChat {
+                    let count = postbox.peerThreadsSummaryTable.get(peerId: peerId)?.effectiveUnreadCount ?? 0
+                    currentReadState = CombinedPeerReadState(states: [(0, .idBased(maxIncomingReadId: 0, maxOutgoingReadId: 1, maxKnownId: 0, count: count, markedUnread: false))])
+                } else {
+                    currentReadState = postbox.readStateTable.getCombinedState(peerId)
+                }
+                
+                var groupIds: [PeerGroupId] = []
+                if let (groupId, _) = self.get(peerId: peerId).includedIndex(peerId: peerId) {
+                    groupIds.append(groupId)
+                }
+                if let groupId = addedToGroupPeerIds[peerId] {
+                    if !groupIds.contains(groupId) {
+                        groupIds.append(groupId)
+                    }
+                }
+                if let groupId = removedFromGroupPeerIds[peerId] {
+                    if !groupIds.contains(groupId) {
+                        groupIds.append(groupId)
+                    }
+                }
+                
+                for groupId in groupIds {
+                    var totalGroupUnreadState: ChatListTotalUnreadState
+                    var summary: PeerGroupUnreadCountersCombinedSummary
+                    if let current = updatedTotalStates[groupId] {
+                        totalGroupUnreadState = current
+                    } else {
+                        totalGroupUnreadState = postbox.messageHistoryMetadataTable.getTotalUnreadState(groupId: groupId)
+                    }
+                    if let current = updatedTotalUnreadSummaries[groupId] {
+                        summary = current
+                    } else {
+                        summary = postbox.groupMessageStatsTable.get(groupId: groupId)
+                    }
+                    
+                    var initialValue: (Int32, Bool, Bool) = (0, false, false)
+                    var currentValue: (Int32, Bool, Bool) = (0, false, false)
+                    
+                    var initialStates: CombinedPeerReadState = CombinedPeerReadState(states: [])
+                    var currentStates: CombinedPeerReadState = CombinedPeerReadState(states: [])
+                    
+                    if addedToGroupPeerIds[peerId] == groupId {
+                        if let currentReadState = currentReadState {
+                            currentValue = (currentReadState.count, currentReadState.isUnread, currentReadState.markedUnread)
+                            currentStates = currentReadState
+                        }
+                    } else if removedFromGroupPeerIds[peerId] == groupId {
+                        if let initialReadState = initialReadState {
+                            initialValue = (initialReadState.count, initialReadState.isUnread, initialReadState.markedUnread)
+                            initialStates = initialReadState
+                        }
+                    } else {
+                        if self.get(peerId: peerId).includedIndex(peerId: peerId)?.0 == groupId {
+                            if let initialReadState = initialReadState {
+                                initialValue = (initialReadState.count, initialReadState.isUnread, initialReadState.markedUnread)
+                                initialStates = initialReadState
+                            }
+                            if let currentReadState = currentReadState {
+                                currentValue = (currentReadState.count, currentReadState.isUnread, currentReadState.markedUnread)
+                                currentStates = currentReadState
+                            }
+                        }
+                    }
+                    
+                    var initialFilteredValue: (Int32, Bool, Bool) = initialValue
+                    var currentFilteredValue: (Int32, Bool, Bool) = currentValue
+                    
+                    if transactionParticipationInTotalUnreadCountUpdates.added.contains(peerId) || transactionParticipationInTotalUnreadCountUpdates.added.contains(notificationPeerId) {
+                        initialFilteredValue = (0, false, false)
+                    } else if transactionParticipationInTotalUnreadCountUpdates.removed.contains(peerId) || transactionParticipationInTotalUnreadCountUpdates.removed.contains(notificationPeerId) {
+                        currentFilteredValue = (0, false, false)
+                    } else {
+                        let isRemovedFromTotalUnreadCount = resolvedIsRemovedFromTotalUnreadCount(globalSettings: globalNotificationSettings, peer: peer, peerSettings: postbox.peerNotificationSettingsTable.getEffective(notificationPeerId))
+                        
+                        if isRemovedFromTotalUnreadCount {
+                            initialFilteredValue = (0, false, false)
+                            currentFilteredValue = (0, false, false)
+                        }
+                    }
+                    
+                    var keptTags: PeerSummaryCounterTags = postbox.seedConfiguration.peerSummaryCounterTags(peer, isContact)
+                    if let (removedTags, addedTags) = updatedPeerTags[peerId] {
+                        keptTags.remove(removedTags)
+                        keptTags.remove(addedTags)
+                        
+                        for tag in removedTags {
+                            alterTags(&totalGroupUnreadState, peerId, tag, { absolute, filtered in
+                                var absolute = absolute
+                                var filtered = filtered
+                                absolute.messageCount -= initialValue.0
+                                if initialValue.1 {
+                                    absolute.chatCount -= 1
+                                }
+                                if initialValue.2 && initialValue.0 == 0 {
+                                    absolute.messageCount -= 1
+                                }
+                                filtered.messageCount -= initialFilteredValue.0
+                                if initialFilteredValue.1 {
+                                    filtered.chatCount -= 1
+                                }
+                                if initialFilteredValue.2 && initialFilteredValue.0 == 0 {
+                                    filtered.messageCount -= 1
+                                }
+                                return (absolute, filtered)
+                            })
+                        }
+                        for tag in addedTags {
+                            alterTags(&totalGroupUnreadState, peerId, tag, { absolute, filtered in
+                                var absolute = absolute
+                                var filtered = filtered
+                                absolute.messageCount += currentValue.0
+                                if currentValue.2 && currentValue.0 == 0 {
+                                    absolute.messageCount += 1
+                                }
+                                if currentValue.1 {
+                                    absolute.chatCount += 1
+                                }
+                                filtered.messageCount += currentFilteredValue.0
+                                if currentFilteredValue.1 {
+                                    filtered.chatCount += 1
+                                }
+                                if currentFilteredValue.2 && currentFilteredValue.0 == 0 {
+                                    filtered.messageCount += 1
+                                }
+                                return (absolute, filtered)
+                            })
+                        }
+                    }
+                    
+                    for tag in keptTags {
+                        alterTags(&totalGroupUnreadState, peerId, tag, { absolute, filtered in
+                            var absolute = absolute
+                            var filtered = filtered
+                            
+                            let chatDifference: Int32
+                            if initialValue.1 != currentValue.1 {
+                                chatDifference = initialValue.1 ? -1 : 1
+                            } else {
+                                chatDifference = 0
+                            }
+                            
+                            let currentUnreadMark: Int32 = currentValue.2 ? 1 : 0
+                            let initialUnreadMark: Int32 = initialValue.2 ? 1 : 0
+                            let messageDifference = max(currentValue.0, currentUnreadMark) - max(initialValue.0, initialUnreadMark)
+                            
+                            let chatFilteredDifference: Int32
+                            if initialFilteredValue.1 != currentFilteredValue.1 {
+                                chatFilteredDifference = initialFilteredValue.1 ? -1 : 1
+                            } else {
+                                chatFilteredDifference = 0
+                            }
+                            let currentFilteredUnreadMark: Int32 = currentFilteredValue.2 ? 1 : 0
+                            let initialFilteredUnreadMark: Int32 = initialFilteredValue.2 ? 1 : 0
+                            let messageFilteredDifference = max(currentFilteredValue.0, currentFilteredUnreadMark) - max(initialFilteredValue.0, initialFilteredUnreadMark)
+                            
+                            absolute.messageCount += messageDifference
+                            absolute.chatCount += chatDifference
+                            filtered.messageCount += messageFilteredDifference
+                            filtered.chatCount += chatFilteredDifference
+                            
+                            return (absolute, filtered)
+                        })
+                    }
+                    
+                    updatedTotalStates[groupId] = totalGroupUnreadState
+                    
+                    var namespaces: [MessageId.Namespace] = []
+                    for (namespace, _) in initialStates.states {
+                        namespaces.append(namespace)
+                    }
+                    for (namespace, _) in currentStates.states {
+                        if !namespaces.contains(namespace) {
+                            namespaces.append(namespace)
+                        }
+                    }
+                    
+                    for namespace in namespaces {
+                        if postbox.seedConfiguration.messageNamespacesRequiringGroupStatsValidation.contains(namespace) && addedToGroupPeerIds[peerId] == groupId && removedFromGroupPeerIds[peerId] == nil {
+                            postbox.synchronizeGroupMessageStatsTable.set(groupId: groupId, namespace: namespace, needsValidation: true, operations: &currentUpdatedGroupSummarySynchronizeOperations)
+                        } else {
+                            var namespaceSummary = summary.namespaces[namespace] ?? PeerGroupUnreadCountersSummary(all: PeerGroupUnreadCounters(messageCount: 0, chatCount: 0))
+                            let previousState = initialStates.states.first(where: { $0.0 == namespace })?.1
+                            let updatedState = currentStates.states.first(where: { $0.0 == namespace })?.1
+                            alterNamespace(summary: &namespaceSummary, previousState: previousState, updatedState: updatedState)
+                            summary.namespaces[namespace] = namespaceSummary
+                        }
+                    }
+                    
+                    updatedTotalUnreadSummaries[groupId] = summary
+                }
+            }
+            
+            for (groupId, state) in updatedTotalStates {
+                if postbox.messageHistoryMetadataTable.getTotalUnreadState(groupId: groupId) != state {
+                    postbox.messageHistoryMetadataTable.setTotalUnreadState(groupId: groupId, state: state)
+                    updatedTotalUnreadStates[groupId] = state
+                }
+            }
+            
+            for groupId in updatedTotalUnreadSummaries.keys {
+                if postbox.groupMessageStatsTable.get(groupId: groupId) != updatedTotalUnreadSummaries[groupId]! {
+                    postbox.groupMessageStatsTable.set(groupId: groupId, summary: updatedTotalUnreadSummaries[groupId]!)
+                    updatedGroupTotalUnreadSummaries[groupId] = updatedTotalUnreadSummaries[groupId]!
+                }
+            }
+        }
+    }
+    
+    override func beforeCommit() {
+        assert(self.updatedPreviousPeerCachedIndices.isEmpty)
+
+        if !self.useCaches {
+            self.cachedPeerIndices.removeAll()
+        }
+    }
+    
+    func debugReindexUnreadCounts(postbox: PostboxImpl, currentTransaction: Transaction) -> ([PeerGroupId: ChatListTotalUnreadState], [PeerGroupId: PeerGroupUnreadCountersCombinedSummary]) {
+        let globalNotificationSettings = postbox.getGlobalNotificationSettings(transaction: currentTransaction)
+        
+        var peerIds: [PeerId] = []
+        for groupId in postbox.chatListTable.existingGroups() + [.root] {
+            let groupPeerIds = postbox.chatListTable.allPeerIds(groupId: groupId)
+            peerIds.append(contentsOf: groupPeerIds)
+        }
+        var totalStates: [PeerGroupId: ChatListTotalUnreadState] = [:]
+        var summaries: [PeerGroupId: PeerGroupUnreadCountersCombinedSummary] = [:]
+        for peerId in peerIds {
+            guard let peer = postbox.peerTable.get(peerId) else {
+                continue
+            }
+            
+            let combinedState: CombinedPeerReadState?
+            if postbox.seedConfiguration.peerSummaryIsThreadBased(peer, peer.associatedPeerId.flatMap(postbox.peerTable.get)).value {
+                let count: Int32 = postbox.peerThreadsSummaryTable.get(peerId: peerId)?.effectiveUnreadCount ?? 0
+                combinedState = CombinedPeerReadState(states: [(0, .idBased(maxIncomingReadId: 0, maxOutgoingReadId: 1, maxKnownId: 0, count: count, markedUnread: false))])
+            } else {
+                combinedState = postbox.readStateTable.getCombinedState(peerId)
+            }
+            
+            guard let combinedState = combinedState else {
+                continue
+            }
+            
+            let isContact = postbox.contactsTable.isContact(peerId: peerId)
+            let notificationPeerId: PeerId
+            if let associatedPeerId = peer.associatedPeerId, peer.associatedPeerOverridesIdentity {
+                notificationPeerId = associatedPeerId
+            } else {
+                notificationPeerId = peerId
+            }
+            let inclusion = self.get(peerId: peerId)
+            if let (groupId, _) = inclusion.includedIndex(peerId: peerId) {
+                if totalStates[groupId] == nil {
+                    totalStates[groupId] = ChatListTotalUnreadState(absoluteCounters: [:], filteredCounters: [:])
+                }
+                
+                let peerMessageCount = combinedState.count
+                
+                let summaryTags = postbox.seedConfiguration.peerSummaryCounterTags(peer, isContact)
+                for tag in summaryTags {
+                    if totalStates[groupId]!.absoluteCounters[tag] == nil {
+                        totalStates[groupId]!.absoluteCounters[tag] = ChatListTotalUnreadCounters(messageCount: 0, chatCount: 0)
+                    }
+                    var messageCount = totalStates[groupId]!.absoluteCounters[tag]!.messageCount
+                    messageCount = messageCount &+ peerMessageCount
+                    if messageCount < 0 {
+                        messageCount = 0
+                    }
+                    if combinedState.isUnread {
+                        totalStates[groupId]!.absoluteCounters[tag]!.chatCount += 1
+                    }
+                    if combinedState.markedUnread {
+                        messageCount = max(1, messageCount)
+                    }
+                    totalStates[groupId]!.absoluteCounters[tag]!.messageCount = messageCount
+                }
+                
+                let isRemovedFromTotalUnreadCount = resolvedIsRemovedFromTotalUnreadCount(globalSettings: globalNotificationSettings, peer: peer, peerSettings: postbox.peerNotificationSettingsTable.getEffective(notificationPeerId))
+                
+                if !isRemovedFromTotalUnreadCount {
+                    for tag in summaryTags {
+                        if totalStates[groupId]!.filteredCounters[tag] == nil {
+                            totalStates[groupId]!.filteredCounters[tag] = ChatListTotalUnreadCounters(messageCount: 0, chatCount: 0)
+                        }
+                        var messageCount = totalStates[groupId]!.filteredCounters[tag]!.messageCount
+                        messageCount = messageCount &+ peerMessageCount
+                        if messageCount < 0 {
+                            messageCount = 0
+                        }
+                        if combinedState.isUnread {
+                            totalStates[groupId]!.filteredCounters[tag]!.chatCount += 1
+                        }
+                        if combinedState.markedUnread {
+                            messageCount = max(1, messageCount)
+                        }
+                        totalStates[groupId]!.filteredCounters[tag]!.messageCount = messageCount
+                    }
+                }
+                
+                for (namespace, state) in combinedState.states {
+                    if summaries[groupId] == nil {
+                        summaries[groupId] = PeerGroupUnreadCountersCombinedSummary(namespaces: [:])
+                    }
+                    if summaries[groupId]!.namespaces[namespace] == nil {
+                        summaries[groupId]!.namespaces[namespace] = PeerGroupUnreadCountersSummary(all: PeerGroupUnreadCounters(messageCount: 0, chatCount: 0))
+                    }
+                    if state.count > 0 {
+                        summaries[groupId]!.namespaces[namespace]!.all.chatCount += 1
+                        summaries[groupId]!.namespaces[namespace]!.all.messageCount += state.count
+                    }
+                }
+            }
+        }
+        
+        return (totalStates, summaries)
+    }
+    
+    func reindexPeerGroupUnreadCounts(postbox: PostboxImpl, groupId: PeerGroupId) -> PeerGroupUnreadCountersCombinedSummary {
+        var summary = PeerGroupUnreadCountersCombinedSummary(namespaces: [:])
+        
+        postbox.chatListTable.forEachPeer(groupId: groupId, { peerId in
+            if peerId.namespace == .max {
+                return
+            }
+            guard let peer = postbox.peerTable.get(peerId) else {
+                return
+            }
+            
+            let combinedState: CombinedPeerReadState?
+            if postbox.seedConfiguration.peerSummaryIsThreadBased(peer, peer.associatedPeerId.flatMap(postbox.peerTable.get)).value {
+                let count: Int32 = postbox.peerThreadsSummaryTable.get(peerId: peerId)?.effectiveUnreadCount ?? 0
+                combinedState = CombinedPeerReadState(states: [(0, .idBased(maxIncomingReadId: 0, maxOutgoingReadId: 1, maxKnownId: 0, count: count, markedUnread: false))])
+            } else {
+                combinedState = postbox.readStateTable.getCombinedState(peerId)
+            }
+            
+            guard let combinedState = combinedState else {
+                return
+            }
+            
+            let inclusion = self.get(peerId: peerId)
+            if let (inclusionGroupId, _) = inclusion.includedIndex(peerId: peerId), inclusionGroupId == groupId {
+                for (namespace, state) in combinedState.states {
+                    if summary.namespaces[namespace] == nil {
+                        summary.namespaces[namespace] = PeerGroupUnreadCountersSummary(all: PeerGroupUnreadCounters(messageCount: 0, chatCount: 0))
+                    }
+                    if state.count > 0 {
+                        summary.namespaces[namespace]!.all.chatCount += 1
+                        summary.namespaces[namespace]!.all.messageCount += state.count
+                    }
+                }
+            }
+        })
+        
+        return summary
+    }
+}

submodules/Postbox/Sources/ChatListIndexView.swift

@@ -0,0 +1,54 @@
+import Foundation
+
+final class MutableChatListIndexView: MutablePostboxView {
+    fileprivate let id: PeerId
+    fileprivate var chatListIndex: ChatListIndex?
+    fileprivate var inclusion: PeerChatListInclusion
+
+    init(postbox: PostboxImpl, id: PeerId) {
+        self.id = id
+        self.chatListIndex = postbox.chatListIndexTable.get(peerId: id).includedIndex(peerId: self.id)?.1
+        self.inclusion = postbox.chatListIndexTable.get(peerId: id).inclusion
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if transaction.currentUpdatedChatListInclusions[self.id] != nil || transaction.currentOperationsByPeerId[self.id] != nil {
+            updated = true
+        }
+
+        if updated {
+            let chatListIndex = postbox.chatListIndexTable.get(peerId: id).includedIndex(peerId: self.id)?.1
+            let inclusion = postbox.chatListIndexTable.get(peerId: id).inclusion
+            if self.chatListIndex != chatListIndex || self.inclusion != inclusion {
+                self.chatListIndex = chatListIndex
+                self.inclusion = inclusion
+                
+                return true
+            } else {
+                return false
+            }
+        } else {
+            return false
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return ChatListIndexView(self)
+    }
+}
+
+public final class ChatListIndexView: PostboxView {
+    public let chatListIndex: ChatListIndex?
+    public let inclusion: PeerChatListInclusion
+    
+    init(_ view: MutableChatListIndexView) {
+        self.chatListIndex = view.chatListIndex
+        self.inclusion = view.inclusion
+    }
+}

submodules/Postbox/Sources/ChatLocation.swift

@@ -0,0 +1,37 @@
+import Foundation
+import SwiftSignalKit
+
+public enum ChatLocationInput {
+    case peer(peerId: PeerId, threadId: Int64?)
+    case thread(peerId: PeerId, threadId: Int64, data: Signal<MessageHistoryViewExternalInput, NoError>)
+    case customChatContents
+}
+
+public extension ChatLocationInput {
+    var peerId: PeerId? {
+        switch self {
+        case let .peer(peerId, _):
+            return peerId
+        case let .thread(peerId, _, _):
+            return peerId
+        case .customChatContents:
+            return nil
+        }
+    }
+    
+    var threadId: Int64? {
+        switch self {
+        case let .peer(_, threadId):
+            return threadId
+        case let .thread(_, threadId, _):
+            return threadId
+        case .customChatContents:
+            return nil
+        }
+    }
+}
+
+public enum ResolvedChatLocationInput {
+    case peer(peerId: PeerId, threadId: Int64?)
+    case external(MessageHistoryViewExternalInput)
+}

submodules/Postbox/Sources/ContactPeerIdsView.swift

@@ -0,0 +1,34 @@
+import Foundation
+
+final class MutableContactPeerIdsView {
+    fileprivate var remoteTotalCount: Int32
+    fileprivate var peerIds: Set<PeerId>
+    
+    init(remoteTotalCount: Int32, peerIds: Set<PeerId>) {
+        self.remoteTotalCount = remoteTotalCount
+        self.peerIds = peerIds
+    }
+    
+    func replay(updateRemoteTotalCount: Int32?, replace replacePeerIds: Set<PeerId>) -> Bool {
+        var updated = false
+        if let updateRemoteTotalCount = updateRemoteTotalCount, self.remoteTotalCount != updateRemoteTotalCount {
+            self.remoteTotalCount = updateRemoteTotalCount
+            updated = true
+        }
+        if self.peerIds != replacePeerIds {
+            self.peerIds = replacePeerIds
+            updated = true
+        }
+        return updated
+    }
+}
+
+public final class ContactPeerIdsView {
+    public let remoteTotalCount: Int32
+    public let peerIds: Set<PeerId>
+    
+    init(_ mutableView: MutableContactPeerIdsView) {
+        self.remoteTotalCount = mutableView.remoteTotalCount
+        self.peerIds = mutableView.peerIds
+    }
+}

submodules/Postbox/Sources/ContactPeersView.swift

@@ -0,0 +1,105 @@
+import Foundation
+
+final class MutableContactPeersView: MutablePostboxView {
+    fileprivate var peers: [PeerId: Peer]
+    fileprivate var peerPresences: [PeerId: PeerPresence]
+    fileprivate var peerIds: Set<PeerId>
+    fileprivate var accountPeer: Peer?
+    private let includePresences: Bool
+    
+    init(postbox: PostboxImpl, accountPeerId: PeerId?, includePresences: Bool) {
+        var peers: [PeerId: Peer] = [:]
+        var peerPresences: [PeerId: PeerPresence] = [:]
+
+        for peerId in postbox.contactsTable.get() {
+            if let peer = postbox.peerTable.get(peerId) {
+                peers[peerId] = peer
+            }
+            if includePresences {
+                if let presence = postbox.peerPresenceTable.get(peerId) {
+                    peerPresences[peerId] = presence
+                }
+            }
+        }
+
+        self.peers = peers
+        self.peerIds = Set<PeerId>(peers.map { $0.0 })
+        self.peerPresences = peerPresences
+        self.accountPeer = accountPeerId.flatMap(postbox.peerTable.get)
+        self.includePresences = includePresences
+    }
+
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        if let replacePeerIds = transaction.replaceContactPeerIds {
+            let removedPeerIds = self.peerIds.subtracting(replacePeerIds)
+            let addedPeerIds = replacePeerIds.subtracting(self.peerIds)
+
+            self.peerIds = replacePeerIds
+
+            for peerId in removedPeerIds {
+                let _ = self.peers.removeValue(forKey: peerId)
+                let _ = self.peerPresences.removeValue(forKey: peerId)
+            }
+
+            for peerId in addedPeerIds {
+                if let peer = postbox.peerTable.get(peerId) {
+                    self.peers[peerId] = peer
+                }
+                if self.includePresences {
+                    if let presence = postbox.peerPresenceTable.get(peerId) {
+                        self.peerPresences[peerId] = presence
+                    }
+                }
+            }
+
+            if !removedPeerIds.isEmpty || !addedPeerIds.isEmpty {
+                updated = true
+            }
+        }
+
+        if self.includePresences, !transaction.currentUpdatedPeerPresences.isEmpty {
+            for peerId in self.peerIds {
+                if let presence = transaction.currentUpdatedPeerPresences[peerId] {
+                    updated = true
+                    self.peerPresences[peerId] = presence
+                }
+            }
+        }
+
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+
+    func immutableView() -> PostboxView {
+        return ContactPeersView(self)
+    }
+}
+
+public final class ContactPeersView: PostboxView {
+    public let peers: [Peer]
+    public let peerPresences: [PeerId: PeerPresence]
+    public let accountPeer: Peer?
+    
+    init(_ mutableView: MutableContactPeersView) {
+        if let accountPeer = mutableView.accountPeer {
+            var peers: [Peer] = []
+            peers.reserveCapacity(mutableView.peers.count)
+            let accountPeerId = accountPeer.id
+            for peer in mutableView.peers.values {
+                if peer.id != accountPeerId {
+                    peers.append(peer)
+                }
+            }
+            self.peers = peers
+        } else {
+            self.peers = mutableView.peers.map({ $0.1 })
+        }
+        self.peerPresences = mutableView.peerPresences
+        self.accountPeer = mutableView.accountPeer
+    }
+}
+

submodules/Postbox/Sources/ContactTable.swift

@@ -0,0 +1,98 @@
+import Foundation
+
+final class ContactTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let peerNameIndexTable: PeerNameIndexTable
+    
+    private var peerIdsBeforeModification: Set<PeerId>?
+    private var peerIds: Set<PeerId>?
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, peerNameIndexTable: PeerNameIndexTable) {
+        self.peerNameIndexTable = peerNameIndexTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(_ id: PeerId, sharedKey: ValueBoxKey = ValueBoxKey(length: 8)) -> ValueBoxKey {
+        sharedKey.setInt64(0, value: id.toInt64())
+        return sharedKey
+    }
+    
+    func isContact(peerId: PeerId) -> Bool {
+        return self.get().contains(peerId)
+    }
+    
+    func get() -> Set<PeerId> {
+        if let peerIds = self.peerIds {
+            return peerIds
+        } else {
+            var peerIds = Set<PeerId>()
+            self.valueBox.scan(self.table, keys: { key in
+                peerIds.insert(PeerId(key.getInt64(0)))
+                return true
+            })
+            self.peerIds = peerIds
+            return peerIds
+        }
+    }
+    
+    func replace(_ ids: Set<PeerId>) {
+        if self.peerIdsBeforeModification == nil {
+           self.peerIdsBeforeModification = self.get()
+        }
+        
+        self.peerIds = ids
+    }
+    
+    override func clearMemoryCache() {
+        assert(self.peerIdsBeforeModification == nil)
+        self.peerIds = nil
+    }
+    
+    func transactionUpdatedPeers() -> [PeerId: (Bool, Bool)] {
+        var result: [PeerId: (Bool, Bool)] = [:]
+        if let peerIdsBeforeModification = self.peerIdsBeforeModification {
+            if let peerIds = self.peerIds {
+                let removedPeerIds = peerIdsBeforeModification.subtracting(peerIds)
+                let addedPeerIds = peerIds.subtracting(peerIdsBeforeModification)
+                
+                for peerId in removedPeerIds.union(addedPeerIds) {
+                    result[peerId] = (removedPeerIds.contains(peerId), addedPeerIds.contains(peerId))
+                }
+            }
+        }
+        return result
+    }
+    
+    override func beforeCommit() {
+        if let peerIdsBeforeModification = self.peerIdsBeforeModification {
+            if let peerIds = self.peerIds {
+                let removedPeerIds = peerIdsBeforeModification.subtracting(peerIds)
+                let addedPeerIds = peerIds.subtracting(peerIdsBeforeModification)
+                
+                let sharedKey = self.key(PeerId(0))
+                
+                for peerId in removedPeerIds {
+                    self.valueBox.remove(self.table, key: self.key(peerId, sharedKey: sharedKey), secure: false)
+                    self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.contacts], includes: false)
+                }
+                
+                for peerId in addedPeerIds {
+                    self.valueBox.set(self.table, key: self.key(peerId, sharedKey: sharedKey), value: MemoryBuffer())
+                    self.peerNameIndexTable.setPeerCategoryState(peerId: peerId, category: [.contacts], includes: true)
+                }
+            } else {
+                assertionFailure()
+            }
+            
+            self.peerIdsBeforeModification = nil
+        }
+
+        if !self.useCaches {
+            self.peerIds = nil
+        }
+    }
+}

submodules/Postbox/Sources/Database.swift

@@ -0,0 +1,81 @@
+//
+// SQLite.swift
+// https://github.com/stephencelis/SQLite.swift
+// Copyright (c) 2014-2015 Stephen Celis.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+//
+
+import Foundation
+import sqlcipher
+
+private let ensureInitialized: Void = {
+    sqlite3_initialize()
+    
+    return Void()
+}()
+
+public final class Database {
+    internal var handle: OpaquePointer? = nil
+
+    public init?(_ location: String, readOnly: Bool) {
+        let _ = ensureInitialized
+        
+        if location != ":memory:" {
+            let _ = open(location + "-guard", O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR)
+        }
+        let flags: Int32
+        if readOnly {
+            flags = SQLITE_OPEN_READONLY
+        } else {
+            flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_FULLMUTEX
+        }
+        let res = sqlite3_open_v2(location, &self.handle, flags, nil)
+        if res != SQLITE_OK {
+            postboxLog("sqlite3_open_v2: \(res)")
+            return nil
+        }
+    }
+
+    deinit {
+        sqlite3_close(self.handle)
+    } // sqlite3_close_v2 in Yosemite/iOS 8?
+
+    public func execute(_ SQL: String) -> Bool {
+        let res = sqlite3_exec(self.handle, SQL, nil, nil, nil)
+        if res == SQLITE_OK {
+            return true
+        } else {
+            if let error = sqlite3_errmsg(self.handle), let str = NSString(utf8String: error) {
+                print("SQL error \(res): \(str) on SQL")
+            } else {
+                print("SQL error \(res) on SQL")
+            }
+            return false
+        }
+    }
+    
+    public func currentError() -> String? {
+        if let error = sqlite3_errmsg(self.handle), let str = NSString(utf8String: error) {
+            return "SQL error \(str)"
+        } else {
+            return nil
+        }
+    }
+}

submodules/Postbox/Sources/DeletedMessagesView.swift

@@ -0,0 +1,51 @@
+import Foundation
+
+final class MutableDeletedMessagesView: MutablePostboxView {
+    let peerId: PeerId
+    var currentDeletedMessages: [MessageId] = []
+
+    init(peerId: PeerId) {
+        self.peerId = peerId
+    }
+
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        if let operations = transaction.currentOperationsByPeerId[self.peerId] {
+            var testMessageIds: [MessageId] = []
+            for operation in operations {
+                switch operation {
+                    case let .Remove(indices):
+                        for (index, _) in indices {
+                            testMessageIds.append(index.id)
+                        }
+                    default:
+                        break
+                }
+            }
+            self.currentDeletedMessages.removeAll()
+            for id in testMessageIds {
+                if !postbox.messageHistoryIndexTable.exists(id) {
+                    self.currentDeletedMessages.append(id)
+                    updated = true
+                }
+            }
+        }
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+
+    func immutableView() -> PostboxView {
+        return DeletedMessagesView(self)
+    }
+}
+
+public final class DeletedMessagesView: PostboxView {
+    public let currentDeletedMessages: [MessageId]
+
+    init(_ view: MutableDeletedMessagesView) {
+        self.currentDeletedMessages = view.currentDeletedMessages
+    }
+}

submodules/Postbox/Sources/DeviceContactImportInfoTable.swift

@@ -0,0 +1,46 @@
+import Foundation
+
+final class DeviceContactImportInfoTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    func get(_ identifier: ValueBoxKey) -> PostboxCoding? {
+        if let value = self.valueBox.get(self.table, key: identifier), let object = PostboxDecoder(buffer: value).decodeRootObject() {
+            return object
+        } else {
+            return nil
+        }
+    }
+    
+    func set(_ identifier: ValueBoxKey, value: PostboxCoding?) {
+        if let value = value {
+            let encoder = PostboxEncoder()
+            encoder.encodeRootObject(value)
+            withExtendedLifetime(encoder, {
+                self.valueBox.set(self.table, key: identifier, value: encoder.readBufferNoCopy())
+            })
+        } else {
+            self.valueBox.remove(self.table, key: identifier, secure: false)
+        }
+    }
+    
+    func getIdentifiers() -> [ValueBoxKey] {
+        var result: [ValueBoxKey] = []
+        self.valueBox.scan(self.table, keys: { key in
+            result.append(key)
+            return true
+        })
+        return result
+    }
+    
+    func enumerateDeviceContactImportInfoItems(_ f: (ValueBoxKey, PostboxCoding) -> Bool) {
+        self.valueBox.scan(self.table, values: { key, value in
+            if let object = PostboxDecoder(buffer: value).decodeRootObject() {
+                return f(key, object)
+            } else {
+                return true
+            }
+        })
+    }
+}

submodules/Postbox/Sources/FailedMessagesView.swift

@@ -0,0 +1,30 @@
+final class MutableFailedMessageIdsView {
+    let peerId: PeerId
+    var ids: Set<MessageId>
+    
+    init(peerId: PeerId, ids: [MessageId]) {
+        self.peerId = peerId
+        self.ids = Set(ids)
+    }
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        let ids = transaction.updatedFailedMessageIds.filter { $0.peerId == self.peerId }
+        let updated = ids != self.ids
+        self.ids = ids
+        return updated
+    }
+    
+    func immutableView() -> FailedMessageIdsView {
+        return FailedMessageIdsView(self.ids)
+    }
+    
+}
+
+
+
+public final class FailedMessageIdsView {
+    public let ids: Set<MessageId>
+
+    fileprivate init(_ ids: Set<MessageId>) {
+        self.ids = ids
+    }
+}

submodules/Postbox/Sources/FileSize.swift

@@ -0,0 +1,33 @@
+import Foundation
+
+// Incuding at least one Objective-C class in a swift file ensures that it doesn't get stripped by the linker
+private final class LinkHelperClass: NSObject {
+}
+
+public func fileSize(_ path: String, useTotalFileAllocatedSize: Bool = false) -> Int64? {
+    /*if useTotalFileAllocatedSize {
+        let url = URL(fileURLWithPath: path)
+        if let values = (try? url.resourceValues(forKeys: Set([.isRegularFileKey, .fileAllocatedSizeKey]))) {
+            if values.isRegularFile ?? false {
+                if let fileSize = values.fileAllocatedSize {
+                    return Int64(fileSize)
+                }
+            }
+        }
+    }*/
+    
+    var value = stat()
+    if lstat(path, &value) == 0 {
+        if (value.st_mode & S_IFMT) == S_IFLNK {
+            return 0
+        }
+        
+        if useTotalFileAllocatedSize {
+            return Int64(value.st_blocks) * Int64(value.st_blksize)
+        }
+        
+        return value.st_size
+    } else {
+        return nil
+    }
+}

submodules/Postbox/Sources/GlobalMessageHistoryTagsTable.swift

@@ -0,0 +1,220 @@
+import Foundation
+
+enum IntermediateGlobalMessageTagsEntry {
+    case message(IntermediateMessage)
+    case hole(MessageIndex)
+    
+    var index: MessageIndex {
+        switch self {
+            case let .message(message):
+                return message.index
+            case let .hole(index):
+                return index
+        }
+    }
+}
+
+enum GlobalMessageHistoryTagsTableEntry {
+    case message(MessageIndex)
+    case hole(MessageIndex)
+    
+    var index: MessageIndex {
+        switch self {
+            case let .message(index):
+                return index
+            case let .hole(index):
+                return index
+        }
+    }
+}
+
+enum GlobalMessageHistoryTagsOperation {
+    case insertMessage(GlobalMessageTags, IntermediateMessage)
+    case insertHole(GlobalMessageTags, MessageIndex)
+    case remove([(GlobalMessageTags, MessageIndex)])
+    case updateTimestamp(GlobalMessageTags, MessageIndex, Int32)
+}
+
+private func parseEntry(key: ValueBoxKey, value: ReadBuffer) -> GlobalMessageHistoryTagsTableEntry {
+    var type: Int8 = 0
+    value.read(&type, offset: 0, length: 1)
+    let index = MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4 + 4)), namespace: key.getInt32(4 + 4), id: key.getInt32(4 + 4 + 4)), timestamp: key.getInt32(4))
+    if type == 0 {
+        return .message(index)
+    } else if type == 1 {
+        return .hole(index)
+    } else {
+        preconditionFailure()
+    }
+}
+
+class GlobalMessageHistoryTagsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 4 + 4 + 4 + 4 + 8)
+    
+    private var cachedInitializedTags = Set<GlobalMessageTags>()
+    
+    private func key(_ tagMask: GlobalMessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 4 + 4 + 4 + 4 + 8)) -> ValueBoxKey {
+        key.setUInt32(0, value: tagMask.rawValue)
+        key.setInt32(4, value: index.timestamp)
+        key.setInt32(4 + 4, value: index.id.namespace)
+        key.setInt32(4 + 4 + 4, value: index.id.id)
+        key.setInt64(4 + 4 + 4 + 4, value: index.id.peerId.toInt64())
+        return key
+    }
+    
+    private func lowerBound(_ tagMask: GlobalMessageTags) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setUInt32(0, value: tagMask.rawValue)
+        return key
+    }
+    
+    private func upperBound(_ tagMask: GlobalMessageTags) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setUInt32(0, value: tagMask.rawValue)
+        return key.successor
+    }
+    
+    func ensureInitialized(_ tagMask: GlobalMessageTags) {
+        for tag in tagMask {
+            if !self.cachedInitializedTags.contains(tag) {
+                var isEmpty = true
+                self.valueBox.range(self.table, start: self.lowerBound(tag), end: self.upperBound(tag), keys: { _ in
+                    isEmpty = false
+                    return false
+                }, limit: 1)
+                if isEmpty {
+                    self.addHole(tag, index: MessageIndex.absoluteUpperBound())
+                }
+                self.cachedInitializedTags.insert(tag)
+            }
+        }
+    }
+    
+    func addMessage(_ tagMask: GlobalMessageTags, index: MessageIndex) -> Bool {
+        self.ensureInitialized(tagMask)
+        
+        assert(tagMask.isSingleTag)
+        var upperIsHole = false
+        self.valueBox.range(self.table, start: self.key(tagMask, index: index, key: self.sharedKey), end: self.upperBound(tagMask), values: { key, value in
+            let entry = parseEntry(key: key, value: value)
+            if case .hole = entry {
+                upperIsHole = true
+            }
+            return false
+        }, limit: 1)
+        if !upperIsHole {
+            var type: Int8 = 0
+            self.valueBox.set(self.table, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer(memory: &type, capacity: 1, length: 1, freeWhenDone: false))
+            return true
+        } else {
+            return false
+        }
+    }
+    
+    func addHole(_ tagMask: GlobalMessageTags, index: MessageIndex) {
+        assert(tagMask.isSingleTag)
+        var type: Int8 = 1
+        self.valueBox.set(self.table, key: self.key(tagMask, index: index, key: self.sharedKey), value: MemoryBuffer(memory: &type, capacity: 1, length: 1, freeWhenDone: false))
+    }
+    
+    func remove(_ tagMask: GlobalMessageTags, index: MessageIndex) {
+        assert(tagMask.isSingleTag)
+        self.valueBox.remove(self.table, key: self.key(tagMask, index: index, key: self.sharedKey), secure: false)
+    }
+    
+    func get(_ tagMask: GlobalMessageTags, index: MessageIndex) -> GlobalMessageHistoryTagsTableEntry? {
+        let key = self.key(tagMask, index: index)
+        if let value = self.valueBox.get(self.table, key: key) {
+            return parseEntry(key: key, value: value)
+        } else {
+            return nil
+        }
+    }
+    
+    func entriesAround(_ tagMask: GlobalMessageTags, index: MessageIndex, count: Int) -> (entries: [GlobalMessageHistoryTagsTableEntry], lower: GlobalMessageHistoryTagsTableEntry?, upper: GlobalMessageHistoryTagsTableEntry?) {
+        var lowerEntries: [GlobalMessageHistoryTagsTableEntry] = []
+        var upperEntries: [GlobalMessageHistoryTagsTableEntry] = []
+        var lower: GlobalMessageHistoryTagsTableEntry?
+        var upper: GlobalMessageHistoryTagsTableEntry?
+        
+        self.valueBox.range(self.table, start: self.key(tagMask, index: index), end: self.lowerBound(tagMask), values: { key, value in
+            lowerEntries.append(parseEntry(key: key, value: value))
+            return true
+        }, limit: count / 2 + 1)
+        
+        if lowerEntries.count >= count / 2 + 1 {
+            lower = lowerEntries.last
+            lowerEntries.removeLast()
+        }
+        
+        self.valueBox.range(self.table, start: self.key(tagMask, index: index).predecessor, end: self.upperBound(tagMask), values: { key, value in
+            upperEntries.append(parseEntry(key: key, value: value))
+            return true
+        }, limit: count - lowerEntries.count + 1)
+        if upperEntries.count >= count - lowerEntries.count + 1 {
+            upper = upperEntries.last
+            upperEntries.removeLast()
+        }
+        
+        if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count {
+            var additionalLowerEntries: [GlobalMessageHistoryTagsTableEntry] = []
+            self.valueBox.range(self.table, start: self.key(tagMask, index: lowerEntries.last!.index), end: self.lowerBound(tagMask), values: { key, value in
+                additionalLowerEntries.append(parseEntry(key: key, value: value))
+                return true
+            }, limit: count - lowerEntries.count - upperEntries.count + 1)
+            if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 {
+                lower = additionalLowerEntries.last
+                additionalLowerEntries.removeLast()
+            }
+            lowerEntries.append(contentsOf: additionalLowerEntries)
+        }
+        
+        var entries: [GlobalMessageHistoryTagsTableEntry] = []
+        entries.append(contentsOf: lowerEntries.reversed())
+        entries.append(contentsOf: upperEntries)
+        return (entries: entries, lower: lower, upper: upper)
+    }
+    
+    func earlierEntries(_ tagMask: GlobalMessageTags, index: MessageIndex?, count: Int) -> [GlobalMessageHistoryTagsTableEntry] {
+        var indices: [GlobalMessageHistoryTagsTableEntry] = []
+        let key: ValueBoxKey
+        if let index = index {
+            key = self.key(tagMask, index: index)
+        } else {
+            key = self.upperBound(tagMask)
+        }
+        self.valueBox.range(self.table, start: key, end: self.lowerBound(tagMask), values: { key, value in
+            indices.append(parseEntry(key: key, value: value))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func laterEntries(_ tagMask: GlobalMessageTags, index: MessageIndex?, count: Int) -> [GlobalMessageHistoryTagsTableEntry] {
+        var indices: [GlobalMessageHistoryTagsTableEntry] = []
+        let key: ValueBoxKey
+        if let index = index {
+            key = self.key(tagMask, index: index)
+        } else {
+            key = self.lowerBound(tagMask)
+        }
+        self.valueBox.range(self.table, start: key, end: self.upperBound(tagMask), values: { key, value in
+            indices.append(parseEntry(key: key, value: value))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func getAll() -> [GlobalMessageHistoryTagsTableEntry] {
+        var indices: [GlobalMessageHistoryTagsTableEntry] = []
+        self.valueBox.scan(self.table, values: { key, value in
+            indices.append(parseEntry(key: key, value: value))
+            return true
+        })
+        return indices
+    }
+}

submodules/Postbox/Sources/GlobalMessageIdsTable.swift

@@ -0,0 +1,51 @@
+import Foundation
+
+final class GlobalMessageIdsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: false)
+    }
+    
+    private let seedConfiguration: SeedConfiguration
+    
+    private let sharedKey = ValueBoxKey(length: 8)
+    private let sharedBuffer = WriteBuffer()
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, seedConfiguration: SeedConfiguration) {
+        self.seedConfiguration = seedConfiguration
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(_ id: Int32) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: Int64(id))
+        return self.sharedKey
+    }
+    
+    func set(_ globalId: Int32, id: MessageId) {
+        assert(id.namespace == 0)
+        assert(id.peerId.namespace._internalGetInt32Value() == 0 || id.peerId.namespace._internalGetInt32Value() == 1)
+        assert(self.seedConfiguration.globalMessageIdsPeerIdNamespaces.contains(GlobalMessageIdsNamespace(peerIdNamespace: id.peerId.namespace, messageIdNamespace: id.namespace)))
+        
+        self.sharedBuffer.reset()
+        var idPeerId: Int64 = id.peerId.toInt64()
+        var idNamespace: Int32 = id.namespace
+        self.sharedBuffer.write(&idPeerId, offset: 0, length: 8)
+        self.sharedBuffer.write(&idNamespace, offset: 0, length: 4)
+        self.valueBox.set(self.table, key: self.key(globalId), value: self.sharedBuffer)
+    }
+    
+    func get(_ globalId: Int32) -> MessageId? {
+        if let value = self.valueBox.get(self.table, key: self.key(globalId)) {
+            var idPeerId: Int64 = 0
+            var idNamespace: Int32 = 0
+            value.read(&idPeerId, offset: 0, length: 8)
+            value.read(&idNamespace, offset: 0, length: 4)
+            return MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: globalId)
+        }
+        return nil
+    }
+    
+    func remove(_ globalId: Int32) {
+        self.valueBox.remove(self.table, key: self.key(globalId), secure: false)
+    }
+}

submodules/Postbox/Sources/GlobalMessageTagsView.swift

@@ -0,0 +1,511 @@
+import Foundation
+
+private enum InternalGlobalMessageTagsEntry: Comparable {
+    case intermediateMessage(IntermediateMessage)
+    case message(Message)
+    case hole(MessageIndex)
+    
+    var index: MessageIndex {
+        switch self {
+            case let .intermediateMessage(message):
+                return message.index
+            case let .message(message):
+                return message.index
+            case let .hole(index):
+                return index
+        }
+    }
+    
+    static func ==(lhs: InternalGlobalMessageTagsEntry, rhs: InternalGlobalMessageTagsEntry) -> Bool {
+        switch lhs {
+            case let .intermediateMessage(lhsMessage):
+                if case let .intermediateMessage(rhsMessage) = rhs {
+                    if lhsMessage.stableVersion != rhsMessage.stableVersion {
+                        return false
+                    }
+                    if lhsMessage.index != rhsMessage.index {
+                        return false
+                    }
+                    return true
+                } else {
+                    return false
+                }
+            case let .message(lhsMessage):
+                if case let .message(rhsMessage) = rhs {
+                    if lhsMessage.stableVersion != rhsMessage.stableVersion {
+                        return false
+                    }
+                    if lhsMessage.index != rhsMessage.index {
+                        return false
+                    }
+                    return true
+                } else {
+                    return false
+                }
+            case let .hole(index):
+                if case .hole(index) = rhs {
+                    return true
+                } else {
+                    return false
+                }
+        }
+    }
+    
+    static func <(lhs: InternalGlobalMessageTagsEntry, rhs: InternalGlobalMessageTagsEntry) -> Bool {
+        return lhs.index < rhs.index
+    }
+}
+
+public enum GlobalMessageTagsEntry {
+    case message(Message)
+    case hole(MessageIndex)
+    
+    public var index: MessageIndex {
+        switch self {
+            case let .message(message):
+                return message.index
+            case let .hole(index):
+                return index
+        }
+    }
+}
+
+final class MutableGlobalMessageTagsViewReplayContext {
+    var invalidEarlier: Bool = false
+    var invalidLater: Bool = false
+    var removedEntries: Bool = false
+    
+    func empty() -> Bool {
+        return !self.removedEntries && !invalidEarlier && !invalidLater
+    }
+}
+
+final class MutableGlobalMessageTagsView: MutablePostboxView {
+    private let globalTag: GlobalMessageTags
+    private let position: MessageIndex
+    private let count: Int
+    private let groupingPredicate: ((Message, Message) -> Bool)?
+    
+    fileprivate var entries: [InternalGlobalMessageTagsEntry]
+    fileprivate var earlier: MessageIndex?
+    fileprivate var later: MessageIndex?
+    
+    init(postbox: PostboxImpl, globalTag: GlobalMessageTags, position: MessageIndex, count: Int, groupingPredicate: ((Message, Message) -> Bool)?) {
+        self.globalTag = globalTag
+        self.position = position
+        self.count = count
+        self.groupingPredicate = groupingPredicate
+        
+        let (entries, lower, upper) = postbox.messageHistoryTable.entriesAround(globalTagMask: globalTag, index: position, count: count)
+        
+        self.entries = entries.map { entry -> InternalGlobalMessageTagsEntry in
+            switch entry {
+                case let .message(message):
+                    return .intermediateMessage(message)
+                case let .hole(index):
+                    return .hole(index)
+            }
+        }
+        self.earlier = lower
+        self.later = upper
+        
+        self.render(postbox: postbox)
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var hasChanges = false
+        
+        let context = MutableGlobalMessageTagsViewReplayContext()
+        
+        var wasSingleHole = false
+        if self.entries.count == 1, case .hole = self.entries[0] {
+            wasSingleHole = true
+        }
+        
+        for operation in transaction.currentGlobalTagsOperations {
+            switch operation {
+            case let .insertMessage(tags, message):
+                if (self.globalTag.rawValue & tags.rawValue) != 0 {
+                    if self.add(.intermediateMessage(message)) {
+                        hasChanges = true
+                    }
+                }
+            case let .insertHole(tags, index):
+                if (self.globalTag.rawValue & tags.rawValue) != 0 {
+                    if self.add(.hole(index)) {
+                        hasChanges = true
+                    }
+                }
+            case let .remove(tagsAndIndices):
+                var indices = Set<MessageIndex>()
+                for (tags, index) in tagsAndIndices {
+                    if (self.globalTag.rawValue & tags.rawValue) != 0 {
+                        indices.insert(index)
+                    }
+                }
+                if !indices.isEmpty {
+                    if self.remove(indices, context: context) {
+                        hasChanges = true
+                    }
+                }
+            case let .updateTimestamp(tags, previousIndex, updatedTimestamp):
+                if (self.globalTag.rawValue & tags.rawValue) != 0 {
+                    inner: for i in 0 ..< self.entries.count {
+                        let entry = self.entries[i]
+                        if entry.index == previousIndex {
+                            let updatedIndex = MessageIndex(id: entry.index.id, timestamp: updatedTimestamp)
+                            if self.remove(Set([entry.index]), context: context) {
+                                hasChanges = true
+                            }
+                            switch entry {
+                            case .hole:
+                                if self.add(.hole(updatedIndex)) {
+                                    hasChanges = true
+                                }
+                            case let .intermediateMessage(message):
+                                if self.add(.intermediateMessage(IntermediateMessage(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, groupingKey: message.groupingKey, groupInfo: message.groupInfo, threadId: message.threadId, timestamp: updatedTimestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, localTags: message.localTags, customTags: message.customTags, forwardInfo: message.forwardInfo, authorId: message.authorId, text: message.text, attributesData: message.attributesData, embeddedMediaData: message.embeddedMediaData, referencedMedia: message.referencedMedia))) {
+                                    hasChanges = true
+                                }
+                            case let .message(message):
+                                if self.add(.message(Message(stableId: message.stableId, stableVersion: message.stableVersion, id: message.id, globallyUniqueId: message.globallyUniqueId, groupingKey: message.groupingKey, groupInfo: message.groupInfo, threadId: message.threadId, timestamp: updatedTimestamp, flags: message.flags, tags: message.tags, globalTags: message.globalTags, localTags: message.localTags, customTags: message.customTags, forwardInfo: message.forwardInfo, author: message.author, text: message.text, attributes: message.attributes, media: message.media, peers: message.peers, associatedMessages: message.associatedMessages, associatedMessageIds: message.associatedMessageIds, associatedMedia: message.associatedMedia, associatedThreadInfo: message.associatedThreadInfo, associatedStories: message.associatedStories))) {
+                                    hasChanges = true
+                                }
+                            }
+                            break inner
+                        }
+                    }
+                }
+            }
+        }
+        
+        if hasChanges || !context.empty() {
+            if wasSingleHole {
+                let (entries, lower, upper) = postbox.messageHistoryTable.entriesAround(globalTagMask: self.globalTag, index: self.position, count: self.count)
+                
+                self.entries = entries.map { entry -> InternalGlobalMessageTagsEntry in
+                    switch entry {
+                    case let .message(message):
+                        return .intermediateMessage(message)
+                    case let .hole(index):
+                        return .hole(index)
+                    }
+                }
+                self.earlier = lower
+                self.later = upper
+            }
+            
+            self.complete(postbox: postbox, context: context)
+            self.render(postbox: postbox)
+            
+            self.render(postbox: postbox)
+        }
+        
+        return hasChanges
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        /*let (entries, lower, upper) = postbox.messageHistoryTable.entriesAround(globalTagMask: globalTag, index: position, count: count)
+
+        self.entries = entries.map { entry -> InternalGlobalMessageTagsEntry in
+            switch entry {
+                case let .message(message):
+                    return .intermediateMessage(message)
+                case let .hole(index):
+                    return .hole(index)
+            }
+        }
+        self.earlier = lower
+        self.later = upper
+
+        self.render(postbox: postbox)
+
+        return true*/
+        return false
+    }
+    
+    private func add(_ entry: InternalGlobalMessageTagsEntry) -> Bool {
+        if self.entries.count == 0 {
+            self.entries.append(entry)
+            return true
+        } else {
+            let first = self.entries[self.entries.count - 1]
+            let last = self.entries[0]
+            
+            let next = self.later
+            
+            if entry.index < last.index {
+                if self.earlier == nil || self.earlier! < entry.index {
+                    if self.entries.count < self.count {
+                        self.entries.insert(entry, at: 0)
+                    } else {
+                        self.earlier = entry.index
+                    }
+                    return true
+                } else {
+                    return false
+                }
+            } else if entry.index > first.index {
+                if next != nil && entry.index > next! {
+                    if self.later == nil || self.later! > entry.index {
+                        if self.entries.count < self.count {
+                            self.entries.append(entry)
+                        } else {
+                            self.later = entry.index
+                        }
+                        return true
+                    } else {
+                        return false
+                    }
+                } else {
+                    self.entries.append(entry)
+                    if self.entries.count > self.count {
+                        self.earlier = self.entries[0].index
+                        self.entries.remove(at: 0)
+                    }
+                    return true
+                }
+            } else if entry != last && entry != first {
+                var i = self.entries.count
+                while i >= 1 {
+                    if self.entries[i - 1].index < entry.index {
+                        break
+                    }
+                    i -= 1
+                }
+                self.entries.insert(entry, at: i)
+                if self.entries.count > self.count {
+                    self.earlier = self.entries[0].index
+                    self.entries.remove(at: 0)
+                }
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    private func remove(_ indices: Set<MessageIndex>, context: MutableGlobalMessageTagsViewReplayContext) -> Bool {
+        var hasChanges = false
+        if let earlier = self.earlier, indices.contains(earlier) {
+            context.invalidEarlier = true
+            hasChanges = true
+        }
+        
+        if let later = self.later , indices.contains(later) {
+            context.invalidLater = true
+            hasChanges = true
+        }
+        
+        if self.entries.count != 0 {
+            var i = self.entries.count - 1
+            while i >= 0 {
+                if indices.contains(self.entries[i].index) {
+                    self.entries.remove(at: i)
+                    context.removedEntries = true
+                    hasChanges = true
+                }
+                i -= 1
+            }
+        }
+        
+        return hasChanges
+    }
+    
+    private func complete(postbox: PostboxImpl, context: MutableGlobalMessageTagsViewReplayContext) {
+        if context.removedEntries {
+            self.completeWithReset(postbox: postbox)
+        } else {
+            if context.invalidEarlier {
+                var earlyId: MessageIndex?
+                let i = 0
+                if i < self.entries.count {
+                    earlyId = self.entries[i].index
+                }
+                
+                let earlierEntries = postbox.messageHistoryTable.earlierEntries(globalTagMask: self.globalTag, index: earlyId, count: 1).map { entry -> InternalGlobalMessageTagsEntry in
+                    switch entry {
+                        case let .message(message):
+                            return .intermediateMessage(message)
+                        case let .hole(index):
+                            return .hole(index)
+                    }
+                }
+                self.earlier = earlierEntries.first?.index
+            }
+            
+            if context.invalidLater {
+                var laterId: MessageIndex?
+                let i = self.entries.count - 1
+                if i >= 0 {
+                    laterId = self.entries[i].index
+                }
+                
+                let laterEntries = postbox.messageHistoryTable.laterEntries(globalTagMask: self.globalTag, index: laterId, count: 1).map { entry -> InternalGlobalMessageTagsEntry in
+                    switch entry {
+                        case let .message(message):
+                            return .intermediateMessage(message)
+                        case let .hole(index):
+                            return .hole(index)
+                    }
+                }
+                self.later = laterEntries.first?.index
+            }
+        }
+    }
+    
+    private func completeWithReset(postbox: PostboxImpl) {
+        var addedEntries: [InternalGlobalMessageTagsEntry] = []
+        
+        var latestAnchor: MessageIndex?
+        if let last = self.entries.last {
+            latestAnchor = last.index
+        }
+        
+        if latestAnchor == nil {
+            if let later = self.later {
+                latestAnchor = later
+            }
+        }
+        
+        if let later = self.later {
+            addedEntries += postbox.messageHistoryTable.laterEntries(globalTagMask: self.globalTag, index: later.globalPredecessor(), count: self.count).map { entry -> InternalGlobalMessageTagsEntry in
+                switch entry {
+                    case let .message(message):
+                        return .intermediateMessage(message)
+                    case let .hole(index):
+                        return .hole(index)
+                }
+            }
+        }
+        if let earlier = self.earlier {
+            addedEntries += postbox.messageHistoryTable.earlierEntries(globalTagMask: self.globalTag, index: earlier.globalSuccessor(), count: self.count).map { entry -> InternalGlobalMessageTagsEntry in
+                switch entry {
+                    case let .message(message):
+                        return .intermediateMessage(message)
+                    case let .hole(index):
+                        return .hole(index)
+                }
+            }
+        }
+        
+        addedEntries += self.entries
+        addedEntries.sort(by: { $0.index < $1.index })
+        var i = addedEntries.count - 1
+        while i >= 1 {
+            if addedEntries[i].index.id == addedEntries[i - 1].index.id {
+                addedEntries.remove(at: i)
+            }
+            i -= 1
+        }
+        self.entries = []
+        
+        var anchorIndex = addedEntries.count - 1
+        if let latestAnchor = latestAnchor {
+            var i = addedEntries.count - 1
+            while i >= 0 {
+                if addedEntries[i].index <= latestAnchor {
+                    anchorIndex = i
+                    break
+                }
+                i -= 1
+            }
+        }
+        
+        /*let indices = self.groupedIndices(addedEntries)
+        
+        if indices.count > self.count {
+            
+        } else {
+            self.later = nil
+            self.earlier = nil
+            self.entries = addedEntries
+        }*/
+        
+        self.later = nil
+        if anchorIndex + 1 < addedEntries.count {
+            self.later = addedEntries[anchorIndex + 1].index
+        }
+        
+        i = anchorIndex
+        while i >= 0 && i > anchorIndex - self.count {
+            self.entries.insert(addedEntries[i], at: 0)
+            i -= 1
+        }
+        
+        self.earlier = nil
+        if anchorIndex - self.count >= 0 {
+            self.earlier = addedEntries[anchorIndex - self.count].index
+        }
+    }
+    
+    private func groupedIndices(_ entries: [InternalGlobalMessageTagsEntry]) -> [[Int]] {
+        if entries.isEmpty {
+            return []
+        }
+        if let groupingPredicate = self.groupingPredicate {
+            var result: [[Int]] = [[0]]
+            for i in 1 ..< entries.count {
+                switch entries[i] {
+                    case .hole:
+                        result.append([i])
+                    case let .message(message):
+                        switch entries[i - 1] {
+                            case .hole:
+                                result.append([i])
+                            case let .message(previousMessage):
+                                if !groupingPredicate(message, previousMessage) {
+                                    result.append([i])
+                                } else {
+                                    result[result.count - 1].append(i)
+                                }
+                            case .intermediateMessage:
+                                assertionFailure()
+                                result.append([i])
+                        }
+                    case .intermediateMessage:
+                        assertionFailure()
+                        result.append([i])
+                }
+            }
+            return result
+        } else {
+            return (0 ..< entries.count).map { [$0] }
+        }
+    }
+    
+    private func render(postbox: PostboxImpl) {
+        for i in 0 ..< self.entries.count {
+            if case let .intermediateMessage(message) = self.entries[i] {
+                self.entries[i] = .message(postbox.renderIntermediateMessage(message))
+            }
+        }
+    }
+    
+    func immutableView() -> PostboxView {
+        return GlobalMessageTagsView(self)
+    }
+}
+
+public final class GlobalMessageTagsView: PostboxView {
+    public let entries: [GlobalMessageTagsEntry]
+    public let earlier: MessageIndex?
+    public let later: MessageIndex?
+    
+    init(_ view: MutableGlobalMessageTagsView) {
+        var entries: [GlobalMessageTagsEntry] = []
+        for entry in view.entries {
+            switch entry {
+                case let .message(message):
+                    entries.append(.message(message))
+                case let .hole(index):
+                    entries.append(.hole(index))
+                case .intermediateMessage:
+                    assertionFailure()
+                    break
+            }
+        }
+        self.entries = entries
+        self.earlier = view.earlier
+        self.later = view.later
+    }
+}

submodules/Postbox/Sources/GroupMessageStatsTable.swift

@@ -0,0 +1,222 @@
+import Foundation
+
+public struct PeerGroupUnreadCounters: PostboxCoding, Equatable {
+    public var messageCount: Int32
+    public var chatCount: Int32
+    
+    public init(messageCount: Int32, chatCount: Int32) {
+        self.messageCount = messageCount
+        self.chatCount = chatCount
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.messageCount = decoder.decodeInt32ForKey("m", orElse: 0)
+        self.chatCount = decoder.decodeInt32ForKey("c", orElse: 0)
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeInt32(self.messageCount, forKey: "m")
+        encoder.encodeInt32(self.chatCount, forKey: "c")
+    }
+}
+
+public struct PeerGroupUnreadCountersSummary: PostboxCoding, Equatable {
+    public var all: PeerGroupUnreadCounters
+    
+    public init(all: PeerGroupUnreadCounters) {
+        self.all = all
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.all = decoder.decodeObjectForKey("a", decoder: { PeerGroupUnreadCounters(decoder: $0) }) as! PeerGroupUnreadCounters
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeObject(self.all, forKey: "a")
+    }
+}
+
+public struct PeerGroupUnreadCountersCombinedSummary: PostboxCoding, Equatable {
+    public enum CountingCategory {
+        case chats
+        case messages
+    }
+    
+    public enum MuteCategory {
+        case all
+    }
+    
+    public var namespaces: [MessageId.Namespace: PeerGroupUnreadCountersSummary]
+    
+    public init(namespaces: [MessageId.Namespace: PeerGroupUnreadCountersSummary]) {
+        self.namespaces = namespaces
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.namespaces = decoder.decodeObjectDictionaryForKey("n", keyDecoder: { $0.decodeInt32ForKey("k", orElse: 0) }, valueDecoder: { PeerGroupUnreadCountersSummary(decoder: $0) })
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeObjectDictionary(self.namespaces, forKey: "n", keyEncoder: { $1.encodeInt32($0, forKey: "k") })
+    }
+    
+    public func count(countingCategory: CountingCategory, mutedCategory: MuteCategory) -> Int32 {
+        var result: Int32 = 0
+        for (_, summary) in self.namespaces {
+            switch mutedCategory {
+                case .all:
+                    switch countingCategory {
+                        case .chats:
+                            result = result &+ summary.all.chatCount
+                        case .messages:
+                            result = result &+ summary.all.messageCount
+                    }
+            }
+        }
+        return result
+    }
+}
+
+public enum ChatListTotalUnreadStateCategory: Int32 {
+    case filtered = 0
+    case raw = 1
+}
+
+public enum ChatListTotalUnreadStateStats: Int32 {
+    case messages = 0
+    case chats = 1
+}
+
+public struct ChatListTotalUnreadState: PostboxCoding, Equatable {
+    public var absoluteCounters: [PeerSummaryCounterTags: ChatListTotalUnreadCounters]
+    public var filteredCounters: [PeerSummaryCounterTags: ChatListTotalUnreadCounters]
+    
+    public init(absoluteCounters: [PeerSummaryCounterTags: ChatListTotalUnreadCounters], filteredCounters: [PeerSummaryCounterTags: ChatListTotalUnreadCounters]) {
+        self.absoluteCounters = absoluteCounters
+        self.filteredCounters = filteredCounters
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.absoluteCounters = decoder.decodeObjectDictionaryForKey("ad", keyDecoder: { decoder in
+            return PeerSummaryCounterTags(rawValue: decoder.decodeInt32ForKey("k", orElse: 0))
+        }, valueDecoder: { decoder in
+            return ChatListTotalUnreadCounters(decoder: decoder)
+        })
+        self.filteredCounters = decoder.decodeObjectDictionaryForKey("fd", keyDecoder: { decoder in
+            return PeerSummaryCounterTags(rawValue: decoder.decodeInt32ForKey("k", orElse: 0))
+        }, valueDecoder: { decoder in
+            return ChatListTotalUnreadCounters(decoder: decoder)
+        })
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeObjectDictionary(self.absoluteCounters, forKey: "ad", keyEncoder: { key, encoder in
+            encoder.encodeInt32(key.rawValue, forKey: "k")
+        })
+        encoder.encodeObjectDictionary(self.filteredCounters, forKey: "fd", keyEncoder: { key, encoder in
+            encoder.encodeInt32(key.rawValue, forKey: "k")
+        })
+    }
+    
+    public func count(for category: ChatListTotalUnreadStateCategory, in statsType: ChatListTotalUnreadStateStats, with tags: PeerSummaryCounterTags) -> Int32 {
+        let counters: [PeerSummaryCounterTags: ChatListTotalUnreadCounters]
+        switch category {
+            case .raw:
+                counters = self.absoluteCounters
+            case .filtered:
+                counters = self.filteredCounters
+        }
+        var result: Int32 = 0
+        for tag in tags {
+            if let category = counters[tag] {
+                switch statsType {
+                    case .messages:
+                        result = result &+ category.messageCount
+                    case .chats:
+                        result = result &+ category.chatCount
+                }
+            }
+        }
+        return result
+    }
+}
+
+final class GroupMessageStatsTable: Table {
+    private var cachedEntries: [PeerGroupId: PeerGroupUnreadCountersCombinedSummary]?
+    private var updatedGroupIds = Set<PeerGroupId>()
+    
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: true)
+    }
+    
+    private func preloadCache() {
+        if self.cachedEntries == nil {
+            var entries: [PeerGroupId: PeerGroupUnreadCountersCombinedSummary] = [:]
+            self.valueBox.scanInt64(self.table, values: { key, value in
+                let groupIdValue: Int32 = Int32(clamping: key)
+                let groupId = PeerGroupId(rawValue: groupIdValue)
+                let state = PeerGroupUnreadCountersCombinedSummary(decoder: PostboxDecoder(buffer: value))
+                entries[groupId] = state
+                return true
+            })
+            self.cachedEntries = entries
+        }
+    }
+    
+    func removeAll() {
+        self.preloadCache()
+        
+        for groupId in self.cachedEntries!.keys {
+            self.set(groupId: groupId, summary: PeerGroupUnreadCountersCombinedSummary(namespaces: [:]))
+        }
+    }
+    
+    func get(groupId: PeerGroupId) -> PeerGroupUnreadCountersCombinedSummary {
+        self.preloadCache()
+        
+        if let state = self.cachedEntries?[groupId] {
+            return state
+        } else {
+            return PeerGroupUnreadCountersCombinedSummary(namespaces: [:])
+        }
+    }
+    
+    func set(groupId: PeerGroupId, summary: PeerGroupUnreadCountersCombinedSummary) {
+        self.preloadCache()
+        
+        let previousSummary = self.get(groupId: groupId)
+        if previousSummary != summary {
+            self.cachedEntries![groupId] = summary
+            self.updatedGroupIds.insert(groupId)
+        }
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedEntries = nil
+        assert(self.updatedGroupIds.isEmpty)
+    }
+    
+    override func beforeCommit() {
+        if !self.updatedGroupIds.isEmpty {
+            if let cachedEntries = self.cachedEntries {
+                let sharedKey = ValueBoxKey(length: 8)
+                let sharedEncoder = PostboxEncoder()
+                for groupId in self.updatedGroupIds {
+                    sharedKey.setInt64(0, value: Int64(groupId.rawValue))
+                    sharedEncoder.reset()
+                    
+                    if let state = cachedEntries[groupId] {
+                        state.encode(sharedEncoder)
+                        self.valueBox.set(self.table, key: sharedKey, value: sharedEncoder.readBufferNoCopy())
+                    } else {
+                        self.valueBox.remove(self.table, key: sharedKey, secure: false)
+                    }
+                }
+            } else {
+                assertionFailure()
+            }
+            
+            self.updatedGroupIds.removeAll()
+        }
+    }
+}

submodules/Postbox/Sources/Hash.swift

@@ -0,0 +1,12 @@
+import Foundation
+import MurMurHash32
+
+public enum HashFunctions {
+    public static func murMurHash32(_ s: String) -> Int32 {
+        return murMurHashString32(s)
+    }
+    
+    public static func murMurHash32(_ d: Data) -> Int32 {
+        return murMurHash32Data(d)
+    }
+}

submodules/Postbox/Sources/HistoryTagInfoView.swift

@@ -0,0 +1,93 @@
+import Foundation
+
+final class MutableHistoryTagInfoView: MutablePostboxView {
+    fileprivate let peerId: PeerId
+    fileprivate let tag: MessageTags
+    
+    fileprivate var currentIndex: MessageIndex?
+    
+    init(postbox: PostboxImpl, peerId: PeerId, tag: MessageTags) {
+        self.peerId = peerId
+        self.tag = tag
+        for namespace in postbox.messageHistoryIndexTable.existingNamespaces(peerId: self.peerId) {
+            if let index = postbox.messageHistoryTagsTable.latestIndex(tag: self.tag, peerId: self.peerId, namespace: namespace) {
+                self.currentIndex = index
+                break
+            }
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if let operations = transaction.currentOperationsByPeerId[self.peerId] {
+            var updated = false
+            var refresh = false
+            for operation in operations {
+                switch operation {
+                case let .InsertMessage(message):
+                    if self.currentIndex == nil {
+                        if message.tags.contains(self.tag) {
+                            self.currentIndex = message.index
+                            updated = true
+                        }
+                    }
+                case let .Remove(indicesAndTags):
+                    if self.currentIndex != nil {
+                        for (index, tags) in indicesAndTags {
+                            if tags.contains(self.tag) {
+                                if index == self.currentIndex {
+                                    self.currentIndex = nil
+                                    updated = true
+                                    refresh = true
+                                }
+                            }
+                        }
+                    }
+                default:
+                    break
+                }
+            }
+            
+            if refresh {
+                for namespace in postbox.messageHistoryIndexTable.existingNamespaces(peerId: self.peerId) {
+                    if let index = postbox.messageHistoryTagsTable.latestIndex(tag: self.tag, peerId: self.peerId, namespace: namespace) {
+                        self.currentIndex = index
+                        break
+                    }
+                }
+            }
+            
+            return updated
+        } else {
+            return false
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        /*var currentIndex: MessageIndex?
+        for namespace in postbox.messageHistoryIndexTable.existingNamespaces(peerId: self.peerId) {
+            if let index = postbox.messageHistoryTagsTable.latestIndex(tag: self.tag, peerId: self.peerId, namespace: namespace) {
+                currentIndex = index
+                break
+            }
+        }
+        if self.currentIndex != currentIndex {
+            self.currentIndex = currentIndex
+            return true
+        } else {
+            return false
+        }*/
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return HistoryTagInfoView(self)
+    }
+}
+
+public final class HistoryTagInfoView: PostboxView {
+    public let isEmpty: Bool
+    
+    init(_ view: MutableHistoryTagInfoView) {
+        self.isEmpty = view.currentIndex == nil
+    }
+}

submodules/Postbox/Sources/InitialMessageHistoryData.swift

@@ -0,0 +1,7 @@
+import Foundation
+
+public struct InitialMessageHistoryData {
+    public let peer: Peer?
+    public let storedInterfaceState: StoredPeerChatInterfaceState?
+    public let associatedMessages: [MessageId: Message]
+}

submodules/Postbox/Sources/IntermediateMessage.swift

@@ -0,0 +1,95 @@
+import Foundation
+
+struct IntermediateMessageForwardInfo {
+    let authorId: PeerId?
+    let sourceId: PeerId?
+    let sourceMessageId: MessageId?
+    let date: Int32
+    let authorSignature: String?
+    let psaType: String?
+    let flags: MessageForwardInfo.Flags
+    
+    init(authorId: PeerId?, sourceId: PeerId?, sourceMessageId: MessageId?, date: Int32, authorSignature: String?, psaType: String?, flags: MessageForwardInfo.Flags) {
+        self.authorId = authorId
+        self.sourceId = sourceId
+        self.sourceMessageId = sourceMessageId
+        self.date = date
+        self.authorSignature = authorSignature
+        self.psaType = psaType
+        self.flags = flags
+    }
+    
+    init(_ storeInfo: StoreMessageForwardInfo) {
+        self.authorId = storeInfo.authorId
+        self.sourceId = storeInfo.sourceId
+        self.sourceMessageId = storeInfo.sourceMessageId
+        self.date = storeInfo.date
+        self.authorSignature = storeInfo.authorSignature
+        self.psaType = storeInfo.psaType
+        self.flags = storeInfo.flags
+    }
+}
+
+class IntermediateMessage {
+    let stableId: UInt32
+    let stableVersion: UInt32
+    let id: MessageId
+    let globallyUniqueId: Int64?
+    let groupingKey: Int64?
+    let groupInfo: MessageGroupInfo?
+    let threadId: Int64?
+    let timestamp: Int32
+    let flags: MessageFlags
+    let tags: MessageTags
+    let globalTags: GlobalMessageTags
+    let localTags: LocalMessageTags
+    let customTags: [MemoryBuffer]
+    let forwardInfo: IntermediateMessageForwardInfo?
+    let authorId: PeerId?
+    let text: String
+    let attributesData: ReadBuffer
+    let embeddedMediaData: ReadBuffer
+    let referencedMedia: [MediaId]
+    
+    var index: MessageIndex {
+        return MessageIndex(id: self.id, timestamp: self.timestamp)
+    }
+    
+    init(stableId: UInt32, stableVersion: UInt32, id: MessageId, globallyUniqueId: Int64?, groupingKey: Int64?, groupInfo: MessageGroupInfo?, threadId: Int64?, timestamp: Int32, flags: MessageFlags, tags: MessageTags, globalTags: GlobalMessageTags, localTags: LocalMessageTags, customTags: [MemoryBuffer], forwardInfo: IntermediateMessageForwardInfo?, authorId: PeerId?, text: String, attributesData: ReadBuffer, embeddedMediaData: ReadBuffer, referencedMedia: [MediaId]) {
+        self.stableId = stableId
+        self.stableVersion = stableVersion
+        self.id = id
+        self.globallyUniqueId = globallyUniqueId
+        self.groupingKey = groupingKey
+        self.groupInfo = groupInfo
+        self.threadId = threadId
+        self.timestamp = timestamp
+        self.flags = flags
+        self.tags = tags
+        self.globalTags = globalTags
+        self.localTags = localTags
+        self.customTags = customTags
+        self.forwardInfo = forwardInfo
+        self.authorId = authorId
+        self.text = text
+        self.attributesData = attributesData
+        self.embeddedMediaData = embeddedMediaData
+        self.referencedMedia = referencedMedia
+    }
+    
+    func withUpdatedTimestamp(_ timestamp: Int32) -> IntermediateMessage {
+        return IntermediateMessage(stableId: self.stableId, stableVersion: self.stableVersion, id: self.id, globallyUniqueId: self.globallyUniqueId, groupingKey: self.groupingKey, groupInfo: self.groupInfo, threadId: self.threadId, timestamp: timestamp, flags: self.flags, tags: self.tags, globalTags: self.globalTags, localTags: self.localTags, customTags: self.customTags, forwardInfo: self.forwardInfo, authorId: self.authorId, text: self.text, attributesData: self.attributesData, embeddedMediaData: self.embeddedMediaData, referencedMedia: self.referencedMedia)
+    }
+    
+    func withUpdatedGroupingKey(_ groupingKey: Int64?) -> IntermediateMessage {
+        return IntermediateMessage(stableId: self.stableId, stableVersion: self.stableVersion, id: self.id, globallyUniqueId: self.globallyUniqueId, groupingKey: groupingKey, groupInfo: self.groupInfo, threadId: self.threadId, timestamp: self.timestamp, flags: self.flags, tags: self.tags, globalTags: self.globalTags, localTags: self.localTags, customTags: self.customTags, forwardInfo: self.forwardInfo, authorId: self.authorId, text: self.text, attributesData: self.attributesData, embeddedMediaData: self.embeddedMediaData, referencedMedia: self.referencedMedia)
+    }
+    
+    func withUpdatedGroupInfo(_ groupInfo: MessageGroupInfo?) -> IntermediateMessage {
+        return IntermediateMessage(stableId: self.stableId, stableVersion: self.stableVersion, id: self.id, globallyUniqueId: self.globallyUniqueId, groupingKey: self.groupingKey, groupInfo: groupInfo, threadId: self.threadId, timestamp: self.timestamp, flags: self.flags, tags: self.tags, globalTags: self.globalTags, localTags: self.localTags, customTags: self.customTags, forwardInfo: self.forwardInfo, authorId: self.authorId, text: self.text, attributesData: self.attributesData, embeddedMediaData: self.embeddedMediaData, referencedMedia: self.referencedMedia)
+    }
+    
+    func withUpdatedEmbeddedMedia(_ embeddedMedia: ReadBuffer) -> IntermediateMessage {
+        return IntermediateMessage(stableId: self.stableId, stableVersion: self.stableVersion, id: self.id, globallyUniqueId: self.globallyUniqueId, groupingKey: self.groupingKey, groupInfo: self.groupInfo, threadId: self.threadId, timestamp: self.timestamp, flags: self.flags, tags: self.tags, globalTags: self.globalTags, localTags: self.localTags, customTags: self.customTags, forwardInfo: self.forwardInfo, authorId: self.authorId, text: self.text, attributesData: self.attributesData, embeddedMediaData: embeddedMedia, referencedMedia: self.referencedMedia)
+    }
+}

submodules/Postbox/Sources/InvalidatedGroupMessageStatsTable.swift

@@ -0,0 +1,57 @@
+import Foundation
+
+public struct PeerGroupAndNamespace: Hashable {
+    public let groupId: PeerGroupId
+    public let namespace: MessageId.Namespace
+    
+    public init(groupId: PeerGroupId, namespace: MessageId.Namespace) {
+        self.groupId = groupId
+        self.namespace = namespace
+    }
+}
+
+final class InvalidatedGroupMessageStatsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8)
+    
+    private func key(groupId: PeerGroupId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        self.sharedKey.setInt32(0, value: groupId.rawValue)
+        self.sharedKey.setInt32(4, value: namespace)
+        return self.sharedKey
+    }
+    
+    private var updatedGroupIds: [PeerGroupAndNamespace: Bool] = [:]
+    
+    func set(groupId: PeerGroupId, namespace: MessageId.Namespace, needsValidation: Bool, operations: inout [PeerGroupAndNamespace: Bool]) {
+        let key = PeerGroupAndNamespace(groupId: groupId, namespace: namespace)
+        self.updatedGroupIds[key] = needsValidation
+        operations[key] = needsValidation
+    }
+    
+    func get() -> Set<PeerGroupAndNamespace> {
+        self.beforeCommit()
+        
+        var result = Set<PeerGroupAndNamespace>()
+        self.valueBox.scan(self.table, keys: { key in
+            result.insert(PeerGroupAndNamespace(groupId: PeerGroupId(rawValue: key.getInt32(0)), namespace: key.getInt32(4)))
+            return true
+        })
+        return result
+    }
+    
+    override func beforeCommit() {
+        if !self.updatedGroupIds.isEmpty {
+            for (groupIdAndNamespace, needsValidation) in self.updatedGroupIds {
+                if needsValidation {
+                    self.valueBox.set(self.table, key: self.key(groupId: groupIdAndNamespace.groupId, namespace: groupIdAndNamespace.namespace), value: MemoryBuffer(data: Data()))
+                } else {
+                    self.valueBox.remove(self.table, key: self.key(groupId: groupIdAndNamespace.groupId, namespace: groupIdAndNamespace.namespace), secure: false)
+                }
+            }
+            self.updatedGroupIds.removeAll()
+        }
+    }
+}

submodules/Postbox/Sources/InvalidatedMessageHistoryTagSummariesView.swift

@@ -0,0 +1,106 @@
+import Foundation
+
+final class MutableInvalidatedMessageHistoryTagSummariesView: MutablePostboxView {
+    private let peerId: PeerId?
+    private let threadId: Int64?
+    private let namespace: MessageId.Namespace
+    private let tagMask: MessageTags
+    
+    var entries = Set<InvalidatedMessageHistoryTagsSummaryEntry>()
+    
+    init(postbox: PostboxImpl, peerId: PeerId?, threadId: Int64?, tagMask: MessageTags, namespace: MessageId.Namespace) {
+        self.peerId = peerId
+        self.threadId = threadId
+        self.tagMask = tagMask
+        self.namespace = namespace
+        
+        self.reload(postbox: postbox)
+    }
+    
+    private func reload(postbox: PostboxImpl) {
+        if let peerId = self.peerId {
+            self.entries.removeAll()
+            self.entries.formUnion(postbox.invalidatedMessageHistoryTagsSummaryTable.getIncludingCustomTags(peerId: peerId, threadId: self.threadId, tagMask: self.tagMask, namespace: self.namespace))
+        } else {
+            for entry in postbox.invalidatedMessageHistoryTagsSummaryTable.get(tagMask: tagMask, namespace: namespace) {
+                self.entries.insert(entry)
+            }
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if let peerId = self.peerId {
+            var maybeUpdated = false
+            loop: for operation in transaction.currentInvalidateMessageTagSummaries {
+                switch operation {
+                case let .add(entry):
+                    if entry.key.peerId == peerId && entry.key.threadId == self.threadId {
+                        maybeUpdated = true
+                        break loop
+                    }
+                case let .remove(key):
+                    if key.peerId == peerId && key.threadId == self.threadId {
+                        maybeUpdated = true
+                        break loop
+                    }
+                }
+            }
+            if maybeUpdated {
+                self.entries.removeAll()
+                self.reload(postbox: postbox)
+                updated = true
+            }
+        } else {
+            for operation in transaction.currentInvalidateMessageTagSummaries {
+                switch operation {
+                case let .add(entry):
+                    if entry.key.namespace == self.namespace && entry.key.tagMask == self.tagMask && entry.key.customTag == nil {
+                        self.entries.insert(entry)
+                        updated = true
+                    }
+                case let .remove(key):
+                    if key.namespace == self.namespace && key.tagMask == self.tagMask && key.customTag == nil {
+                        for entry in self.entries {
+                            if entry.key == key {
+                                self.entries.remove(entry)
+                                break
+                            }
+                        }
+                        updated = true
+                    }
+                }
+            }
+        }
+        
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        /*var entries = Set<InvalidatedMessageHistoryTagsSummaryEntry>()
+        for entry in postbox.invalidatedMessageHistoryTagsSummaryTable.get(tagMask: tagMask, namespace: namespace) {
+            entries.insert(entry)
+        }
+        if self.entries != entries {
+            self.entries = entries
+            return true
+        } else {
+            return false
+        }*/
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return InvalidatedMessageHistoryTagSummariesView(self)
+    }
+}
+
+public final class InvalidatedMessageHistoryTagSummariesView: PostboxView {
+    public let entries: Set<InvalidatedMessageHistoryTagsSummaryEntry>
+    
+    init(_ view: MutableInvalidatedMessageHistoryTagSummariesView) {
+        self.entries = view.entries
+    }
+}
+

submodules/Postbox/Sources/InvalidatedMessageHistoryTagsSummaryTable.swift

@@ -0,0 +1,236 @@
+import Foundation
+
+public struct InvalidatedMessageHistoryTagsSummaryKey: Comparable, Hashable {
+    public let peerId: PeerId
+    public let namespace: MessageId.Namespace
+    public let tagMask: MessageTags
+    public let threadId: Int64?
+    public let customTag: MemoryBuffer?
+    
+    public init(peerId: PeerId, namespace: MessageId.Namespace, tagMask: MessageTags, threadId: Int64?, customTag: MemoryBuffer?) {
+        self.peerId = peerId
+        self.namespace = namespace
+        self.tagMask = tagMask
+        self.threadId = threadId
+        self.customTag = customTag
+    }
+    
+    public static func <(lhs: InvalidatedMessageHistoryTagsSummaryKey, rhs: InvalidatedMessageHistoryTagsSummaryKey) -> Bool {
+        if lhs.peerId != rhs.peerId {
+            return lhs.peerId < rhs.peerId
+        }
+        if lhs.namespace != rhs.namespace {
+            return lhs.namespace != rhs.namespace
+        }
+        if lhs.tagMask != rhs.tagMask {
+            return lhs.tagMask.rawValue < rhs.tagMask.rawValue
+        }
+        if let lhsThreadId = lhs.threadId, let rhsThreadId = rhs.threadId {
+            if lhsThreadId != rhsThreadId {
+                return lhsThreadId < rhsThreadId
+            }
+        } else if (lhs.threadId == nil) != (rhs.threadId == nil) {
+            if lhs.threadId != nil {
+                return true
+            } else {
+                return false
+            }
+        }
+        if let lhsCustomTag = lhs.customTag, let rhsCustomTag = rhs.customTag {
+            if lhsCustomTag != rhsCustomTag {
+                return lhsCustomTag < rhsCustomTag
+            }
+        } else if (lhs.customTag == nil) != (rhs.customTag == nil) {
+            if lhs.customTag != nil {
+                return true
+            } else {
+                return false
+            }
+        }
+        return false
+    }
+}
+
+public struct InvalidatedMessageHistoryTagsSummaryEntry: Equatable, Hashable {
+    public let key: InvalidatedMessageHistoryTagsSummaryKey
+    public let version: Int32
+    
+    public init(key: InvalidatedMessageHistoryTagsSummaryKey, version: Int32) {
+        self.key = key
+        self.version = version
+    }
+}
+
+enum InvalidatedMessageHistoryTagsSummaryEntryOperation {
+    case add(InvalidatedMessageHistoryTagsSummaryEntry)
+    case remove(InvalidatedMessageHistoryTagsSummaryKey)
+}
+
+final class InvalidatedMessageHistoryTagsSummaryTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private func key(_ key: InvalidatedMessageHistoryTagsSummaryKey) -> ValueBoxKey {
+        if let customTag = key.customTag, customTag.length != 0 {
+            var keyLength = 4 + 4 + 8
+            keyLength += 8
+            keyLength += customTag.length
+            let result = ValueBoxKey(length: keyLength)
+            
+            var offset = 0
+            result.setUInt32(offset, value: key.tagMask.rawValue)
+            offset += 4
+            
+            result.setInt32(offset, value: key.namespace)
+            offset += 4
+            
+            result.setInt64(offset, value: key.peerId.toInt64())
+            offset += 8
+            
+            result.setInt64(offset, value: key.threadId ?? 0)
+            offset += 8
+            
+            if customTag.length != 0 {
+                customTag.withRawBufferPointer { buffer in
+                    result.setBytes(offset, value: buffer)
+                    offset += buffer.count
+                }
+            }
+            
+            return result
+        } else if let threadId = key.threadId {
+            let result = ValueBoxKey(length: 4 + 4 + 8 + 8)
+            result.setUInt32(0, value: key.tagMask.rawValue)
+            result.setInt32(4, value: key.namespace)
+            result.setInt64(4 + 4, value: key.peerId.toInt64())
+            result.setInt64(4 + 4 + 8, value: threadId)
+            return result
+        } else {
+            let result = ValueBoxKey(length: 4 + 4 + 8)
+            result.setUInt32(0, value: key.tagMask.rawValue)
+            result.setInt32(4, value: key.namespace)
+            result.setInt64(4 + 4, value: key.peerId.toInt64())
+            return result
+        }
+    }
+    
+    private func lowerBound(tagMask: MessageTags, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let result = ValueBoxKey(length: 4 + 4)
+        result.setUInt32(0, value: tagMask.rawValue)
+        result.setInt32(4, value: namespace)
+        return result
+    }
+    
+    private func upperBound(tagMask: MessageTags, namespace: MessageId.Namespace) -> ValueBoxKey {
+        return self.lowerBound(tagMask: tagMask, namespace: namespace).successor
+    }
+    
+    func get(tagMask: MessageTags, namespace: MessageId.Namespace) -> [InvalidatedMessageHistoryTagsSummaryEntry] {
+        var entries: [InvalidatedMessageHistoryTagsSummaryEntry] = []
+        self.valueBox.range(self.table, start: self.lowerBound(tagMask: tagMask, namespace: namespace), end: self.upperBound(tagMask: tagMask, namespace: namespace), values: { key, value in
+            var version: Int32 = 0
+            value.read(&version, offset: 0, length: 4)
+            
+            var threadId: Int64?
+            var customTag: MemoryBuffer?
+            if key.length >= 4 + 4 + 8 + 8 {
+                threadId = key.getInt64(4 + 4 + 8)
+                
+                if key.length > 4 + 4 + 8 + 8 {
+                    customTag = key.getMemoryBuffer(4 + 4 + 8 + 8, length: key.length - (4 + 4 + 8 + 8))
+                    if threadId == 0 {
+                        threadId = nil
+                    }
+                }
+            }
+            
+            entries.append(InvalidatedMessageHistoryTagsSummaryEntry(key: InvalidatedMessageHistoryTagsSummaryKey(peerId: PeerId(key.getInt64(4 + 4)), namespace: key.getInt32(4), tagMask: MessageTags(rawValue: key.getUInt32(0)), threadId: threadId, customTag: customTag), version: version))
+            return true
+        }, limit: 0)
+        return entries
+    }
+    
+    func get(peerId: PeerId, threadId: Int64?, tagMask: MessageTags, namespace: MessageId.Namespace, customTag: MemoryBuffer?) -> InvalidatedMessageHistoryTagsSummaryEntry? {
+        return self.get(InvalidatedMessageHistoryTagsSummaryKey(peerId: peerId, namespace: namespace, tagMask: tagMask, threadId: threadId, customTag: customTag))
+    }
+    
+    func getIncludingCustomTags(peerId: PeerId, threadId: Int64?, tagMask: MessageTags, namespace: MessageId.Namespace) -> [InvalidatedMessageHistoryTagsSummaryEntry] {
+        var entries: [InvalidatedMessageHistoryTagsSummaryEntry] = []
+        
+        let peerKey = self.key(InvalidatedMessageHistoryTagsSummaryKey(peerId: peerId, namespace: namespace, tagMask: tagMask, threadId: threadId, customTag: nil))
+        self.valueBox.range(
+            self.table,
+            start: peerKey.predecessor, end: peerKey.successor, values: { key, value in
+                var version: Int32 = 0
+                value.read(&version, offset: 0, length: 4)
+                
+                var threadId: Int64?
+                var customTag: MemoryBuffer?
+                if key.length >= 4 + 4 + 8 + 8 {
+                    threadId = key.getInt64(4 + 4 + 8)
+                    
+                    if key.length > 4 + 4 + 8 + 8 {
+                        customTag = key.getMemoryBuffer(4 + 4 + 8 + 8, length: key.length - (4 + 4 + 8 + 8))
+                        if threadId == 0 {
+                            threadId = nil
+                        }
+                    }
+                }
+                
+                let entry = InvalidatedMessageHistoryTagsSummaryEntry(key: InvalidatedMessageHistoryTagsSummaryKey(peerId: PeerId(key.getInt64(4 + 4)), namespace: key.getInt32(4), tagMask: MessageTags(rawValue: key.getUInt32(0)), threadId: threadId, customTag: customTag), version: version)
+                assert(entry.key.peerId == peerId)
+                assert(entry.key.namespace == namespace)
+                assert(entry.key.tagMask == tagMask)
+                assert(entry.key.threadId == threadId)
+                
+                entries.append(entry)
+                
+                return true
+            },
+            limit: 0
+        )
+        
+        return entries
+    }
+    
+    private func get(_ key: InvalidatedMessageHistoryTagsSummaryKey) -> InvalidatedMessageHistoryTagsSummaryEntry? {
+        if let value = self.valueBox.get(self.table, key: self.key(key)) {
+            var version: Int32 = 0
+            value.read(&version, offset: 0, length: 4)
+            return InvalidatedMessageHistoryTagsSummaryEntry(key: key, version: version)
+        } else {
+            return nil
+        }
+    }
+    
+    func insert(_ key: InvalidatedMessageHistoryTagsSummaryKey, operations: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        var version: Int32 = 0
+        if let entry = self.get(key) {
+            self.remove(entry, operations: &operations)
+            version = entry.version + 1
+        }
+        self.valueBox.set(self.table, key: self.key(key), value: MemoryBuffer(memory: &version, capacity: 4, length: 4, freeWhenDone: false))
+        operations.append(.add(InvalidatedMessageHistoryTagsSummaryEntry(key: key, version: version)))
+    }
+    
+    func remove(_ entry: InvalidatedMessageHistoryTagsSummaryEntry, operations: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        if let current = self.get(entry.key), current.version == entry.version {
+            self.valueBox.remove(self.table, key: self.key(entry.key), secure: false)
+            operations.append(.remove(entry.key))
+        }
+    }
+    
+    func removeEntriesWithCustomTags(peerId: PeerId, threadId: Int64?, namespace: MessageId.Namespace, tagMask: MessageTags, operations: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        for entry in self.getIncludingCustomTags(peerId: peerId, threadId: threadId, tagMask: tagMask, namespace: namespace) {
+            self.valueBox.remove(self.table, key: self.key(entry.key), secure: false)
+            operations.append(.remove(entry.key))
+        }
+    }
+    
+    override func clearMemoryCache() {
+    }
+    
+    override func beforeCommit() {
+    }
+}

submodules/Postbox/Sources/IpcPipe.swift

@@ -0,0 +1,55 @@
+import Foundation
+import SwiftSignalKit
+
+func ipcNotify(basePath: String, data: Int64) {
+    DispatchQueue.global(qos: .default).async {
+        let path = basePath + ".ipc"
+        let fd = open(path, open(path, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR))
+        if fd != -1 {
+            var value = data
+            write(fd, &value, 8)
+            close(fd)
+        }
+    }
+}
+
+func ipcNotifications(basePath: String) -> Signal<Int64, Void> {
+    return Signal { subscriber in
+        let queue = Queue()
+        let disposable = MetaDisposable()
+        
+        queue.async {
+            let path = basePath + ".ipc"
+            let fd = open(path, open(path, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR))
+            if fd != -1 {
+                let readSource = DispatchSource.makeFileSystemObjectSource(fileDescriptor: fd, eventMask: [.write])
+                
+                readSource.setEventHandler(handler: {
+                    subscriber.putNext(Int64.max)
+                    /*lseek(fd, 0, SEEK_SET)
+                    var value: Int64 = 0
+                    if read(fd, &value, 8) == 8 {
+                        if previousValue != value {
+                            previousValue = value
+                            subscriber.putNext(value)
+                        }
+                    }*/
+                })
+                
+                readSource.resume()
+                
+                disposable.set(ActionDisposable {
+                    queue.async {
+                        readSource.cancel()
+                        close(fd)
+                    }
+                })
+            } else {
+                subscriber.putError(Void())
+            }
+        }
+        
+        return disposable
+    }
+}
+

submodules/Postbox/Sources/IsContactView.swift

@@ -0,0 +1,43 @@
+import Foundation
+
+final class MutableIsContactView: MutablePostboxView {
+    fileprivate let id: PeerId
+    fileprivate var isContact: Bool
+
+    init(postbox: PostboxImpl, id: PeerId) {
+        self.id = id
+        self.isContact = postbox.contactsTable.isContact(peerId: self.id)
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        if transaction.replaceContactPeerIds != nil {
+            let isContact = postbox.contactsTable.isContact(peerId: self.id)
+            if self.isContact != isContact {
+                self.isContact = isContact
+                updated = true
+            }
+        }
+        if updated {
+            return true
+        } else {
+            return false
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return IsContactView(self)
+    }
+}
+
+public final class IsContactView: PostboxView {
+    public let isContact: Bool
+    
+    init(_ view: MutableIsContactView) {
+        self.isContact = view.isContact
+    }
+}

submodules/Postbox/Sources/ItemCacheMetaTable.swift

@@ -0,0 +1,72 @@
+import Foundation
+
+public typealias ItemCacheCollectionId = Int8
+
+struct ItemCacheCollectionState: PostboxCoding {
+    let nextAccessIndex: Int32
+    
+    init(decoder: PostboxDecoder) {
+        self.nextAccessIndex = decoder.decodeInt32ForKey("i", orElse: 0)
+    }
+    
+    func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeInt32(self.nextAccessIndex, forKey: "i")
+    }
+}
+
+final class ItemCacheMetaTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: false)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8)
+    
+    private var cachedCollectionStates: [ItemCacheCollectionId: ItemCacheCollectionState] = [:]
+    private var updatedCollectionStateIds = Set<ItemCacheCollectionId>()
+    
+    private func key(_ id: ItemCacheCollectionId) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: Int64(id))
+        return self.sharedKey
+    }
+    
+    private func get(_ id: ItemCacheCollectionId) -> ItemCacheCollectionState? {
+        if let cached = self.cachedCollectionStates[id] {
+            return cached
+        } else {
+            if let value = self.valueBox.get(self.table, key: self.key(id)), let state = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCacheCollectionState {
+                self.cachedCollectionStates[id] = state
+                return state
+            } else {
+                return nil
+            }
+        }
+    }
+    
+    private func set(_ id: ItemCacheCollectionId, state: ItemCacheCollectionState) {
+        self.cachedCollectionStates[id] = state
+        self.updatedCollectionStateIds.insert(id)
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedCollectionStates.removeAll()
+        self.updatedCollectionStateIds.removeAll()
+    }
+    
+    override func beforeCommit() {
+        if !self.updatedCollectionStateIds.isEmpty {
+            let sharedEncoder = PostboxEncoder()
+            for id in self.updatedCollectionStateIds {
+                if let state = self.cachedCollectionStates[id] {
+                    sharedEncoder.reset()
+                    sharedEncoder.encodeRootObject(state)
+                    withExtendedLifetime(sharedEncoder, {
+                        self.valueBox.set(self.table, key: self.key(id), value: sharedEncoder.readBufferNoCopy())
+                    })
+                }
+            }
+        }
+        
+        self.updatedCollectionStateIds.removeAll()
+    }
+}
+

submodules/Postbox/Sources/ItemCacheTable.swift

@@ -0,0 +1,94 @@
+import Foundation
+
+public final class ItemCacheEntryId: Equatable, Hashable {
+    public let collectionId: ItemCacheCollectionId
+    public let key: ValueBoxKey
+    
+    public init(collectionId: ItemCacheCollectionId, key: ValueBoxKey) {
+        self.collectionId = collectionId
+        self.key = key
+    }
+
+    public func hash(into hasher: inout Hasher) {
+        hasher.combine(self.collectionId)
+        hasher.combine(self.key)
+    }
+
+    public static func ==(lhs: ItemCacheEntryId, rhs: ItemCacheEntryId) -> Bool {
+        return lhs.collectionId == rhs.collectionId && lhs.key == rhs.key
+    }
+}
+
+private enum ItemCacheSection: Int8 {
+    case items = 0
+    case accessIndexToItemId = 1
+    case itemIdToAccessIndex = 2
+}
+
+final class ItemCacheTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: false)
+    }
+    
+    private func itemKey(id: ItemCacheEntryId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1 + 1 + id.key.length)
+        key.setInt8(0, value: ItemCacheSection.items.rawValue)
+        key.setInt8(1, value: id.collectionId)
+        memcpy(key.memory.advanced(by: 2), id.key.memory, id.key.length)
+        return key
+    }
+    
+    private func lowerBound(collectionId: ItemCacheCollectionId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1 + 1)
+        key.setInt8(0, value: ItemCacheSection.items.rawValue)
+        key.setInt8(1, value: collectionId)
+        return key
+    }
+    
+    private func upperBound(collectionId: ItemCacheCollectionId) -> ValueBoxKey {
+        return self.lowerBound(collectionId: collectionId).successor
+    }
+    
+    private func itemIdToAccessIndexKey(id: ItemCacheEntryId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1 + 1 + id.key.length)
+        key.setInt8(0, value: ItemCacheSection.accessIndexToItemId.rawValue)
+        key.setInt8(1, value: id.collectionId)
+        memcpy(key.memory.advanced(by: 2), id.key.memory, id.key.length)
+        return key
+    }
+    
+    private func accessIndexToItemId(collectionId: ItemCacheCollectionId, index: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1 + 1 + 4)
+        key.setInt8(0, value: ItemCacheSection.accessIndexToItemId.rawValue)
+        key.setInt8(1, value: collectionId)
+        key.setInt32(2, value: index)
+        return key
+    }
+
+    func put(id: ItemCacheEntryId, entry: CodableEntry, metaTable: ItemCacheMetaTable) {
+        self.valueBox.set(self.table, key: self.itemKey(id: id), value: ReadBuffer(data: entry.data))
+    }
+
+    func retrieve(id: ItemCacheEntryId, metaTable: ItemCacheMetaTable) -> CodableEntry? {
+        if let value = self.valueBox.get(self.table, key: self.itemKey(id: id)) {
+            return CodableEntry(data: value.makeData())
+        }
+        return nil
+    }
+    
+    func remove(id: ItemCacheEntryId, metaTable: ItemCacheMetaTable) {
+        self.valueBox.remove(self.table, key: self.itemKey(id: id), secure: false)
+    }
+    
+    func removeAll(collectionId: ItemCacheCollectionId) {
+        self.valueBox.removeRange(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId))
+    }
+    
+    override func clearMemoryCache() {
+        
+    }
+    
+    override func beforeCommit() {
+        
+    }
+}

submodules/Postbox/Sources/ItemCollection.swift

@@ -0,0 +1,103 @@
+import Foundation
+
+public struct ItemCollectionId: Comparable, Hashable, Codable {
+    public typealias Namespace = Int32
+    public typealias Id = Int64
+    
+    public let namespace: ItemCollectionId.Namespace
+    public let id: ItemCollectionId.Id
+    
+    public init(namespace: ItemCollectionId.Namespace, id: ItemCollectionId.Id) {
+        self.namespace = namespace
+        self.id = id
+    }
+    
+    public static func ==(lhs: ItemCollectionId, rhs: ItemCollectionId) -> Bool {
+        return lhs.namespace == rhs.namespace && lhs.id == rhs.id
+    }
+    
+    public static func <(lhs: ItemCollectionId, rhs: ItemCollectionId) -> Bool {
+        if lhs.namespace == rhs.namespace {
+            return lhs.id < rhs.id
+        } else {
+            return lhs.namespace < rhs.namespace
+        }
+    }
+    
+    public static func encodeArrayToBuffer(_ array: [ItemCollectionId], buffer: WriteBuffer) {
+        var length: Int32 = Int32(array.count)
+        buffer.write(&length, offset: 0, length: 4)
+        for id in array {
+            var idNamespace = id.namespace
+            buffer.write(&idNamespace, offset: 0, length: 4)
+            var idId = id.id
+            buffer.write(&idId, offset: 0, length: 8)
+        }
+    }
+    
+    public static func decodeArrayFromBuffer(_ buffer: ReadBuffer) -> [ItemCollectionId] {
+        var length: Int32 = 0
+        memcpy(&length, buffer.memory, 4)
+        buffer.offset += 4
+        var i = 0
+        var array: [ItemCollectionId] = []
+        array.reserveCapacity(Int(length))
+        while i < Int(length) {
+            var idNamespace: Int32 = 0
+            buffer.read(&idNamespace, offset: 0, length: 4)
+            var idId: Int64 = 0
+            buffer.read(&idId, offset: 0, length: 8)
+            array.append(ItemCollectionId(namespace: idNamespace, id: idId))
+            i += 1
+        }
+        return array
+    }
+}
+
+public protocol ItemCollectionInfo: PostboxCoding {
+    
+}
+
+public struct ItemCollectionItemIndex: Comparable, Hashable {
+    public typealias Index = Int32
+    public typealias Id = Int64
+    
+    public let index: ItemCollectionItemIndex.Index
+    public let id: ItemCollectionItemIndex.Id
+    
+    public init(index: ItemCollectionItemIndex.Index, id: ItemCollectionItemIndex.Id) {
+        self.index = index
+        self.id = id
+    }
+    
+    public static func ==(lhs: ItemCollectionItemIndex, rhs: ItemCollectionItemIndex) -> Bool {
+        return lhs.index == rhs.index && lhs.id == rhs.id
+    }
+    
+    public static func <(lhs: ItemCollectionItemIndex, rhs: ItemCollectionItemIndex) -> Bool {
+        if lhs.index == rhs.index {
+            return lhs.id < rhs.id
+        } else {
+            return lhs.index < rhs.index
+        }
+    }
+    
+    static var lowerBound: ItemCollectionItemIndex {
+        return ItemCollectionItemIndex(index: 0, id: 0)
+    }
+    
+    static var upperBound: ItemCollectionItemIndex {
+        return ItemCollectionItemIndex(index: Int32.max, id: Int64.max)
+    }
+}
+
+public protocol ItemCollectionItem: PostboxCoding {
+    var index: ItemCollectionItemIndex { get }
+    var indexKeys: [MemoryBuffer] { get }
+}
+
+public enum ItemCollectionSearchQuery {
+    case exact(ValueBoxKey)
+    case matching([ValueBoxKey])
+    case any([ValueBoxKey])
+}

submodules/Postbox/Sources/ItemCollectionIdsView.swift

@@ -0,0 +1,70 @@
+import Foundation
+
+final class MutableItemCollectionIdsView: MutablePostboxView {
+    let namespaces: [ItemCollectionId.Namespace]
+    var idsByNamespace: [ItemCollectionId.Namespace: Set<ItemCollectionId>]
+    
+    init(postbox: PostboxImpl, namespaces: [ItemCollectionId.Namespace]) {
+        self.namespaces = namespaces
+        
+        var idsByNamespace: [ItemCollectionId.Namespace: Set<ItemCollectionId>] = [:]
+        for namespace in namespaces {
+            let ids = postbox.itemCollectionInfoTable.getIds(namespace: namespace)
+            idsByNamespace[namespace] = Set(ids)
+        }
+        self.idsByNamespace = idsByNamespace
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if transaction.currentItemCollectionInfosOperations.isEmpty {
+            return false
+        }
+        
+        var updated = false
+        
+        var reloadInfosNamespaces = Set<ItemCollectionId.Namespace>()
+        for operation in transaction.currentItemCollectionInfosOperations {
+            switch operation {
+                case let .replaceInfos(namespace):
+                    reloadInfosNamespaces.insert(namespace)
+            }
+        }
+        if !reloadInfosNamespaces.isEmpty {
+            for namespace in self.namespaces {
+                if reloadInfosNamespaces.contains(namespace) {
+                    updated = true
+                    let ids = postbox.itemCollectionInfoTable.getIds(namespace: namespace)
+                    self.idsByNamespace[namespace] = Set(ids)
+                }
+            }
+        }
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        /*var idsByNamespace: [ItemCollectionId.Namespace: Set<ItemCollectionId>] = [:]
+        for namespace in namespaces {
+            let ids = postbox.itemCollectionInfoTable.getIds(namespace: namespace)
+            idsByNamespace[namespace] = Set(ids)
+        }
+        if self.idsByNamespace != idsByNamespace {
+            self.idsByNamespace = idsByNamespace
+            return true
+        } else {
+            return false
+        }*/
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return ItemCollectionIdsView(self)
+    }
+}
+
+public final class ItemCollectionIdsView: PostboxView {
+    public let idsByNamespace: [ItemCollectionId.Namespace: Set<ItemCollectionId>]
+    
+    init(_ view: MutableItemCollectionIdsView) {
+        self.idsByNamespace = view.idsByNamespace
+    }
+}

submodules/Postbox/Sources/ItemCollectionInfoTable.swift

@@ -0,0 +1,172 @@
+import Foundation
+
+enum ItemCollectionInfosOperation {
+    case replaceInfos(ItemCollectionId.Namespace)
+}
+
+final class ItemCollectionInfoTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: false)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 4 + 4 + 8)
+    
+    private var cachedInfos: [ItemCollectionId.Namespace: [(Int, ItemCollectionId, ItemCollectionInfo)]] = [:]
+    
+    private func key(collectionId: ItemCollectionId, index: Int32) -> ValueBoxKey {
+        self.sharedKey.setInt32(0, value: collectionId.namespace)
+        self.sharedKey.setInt32(4, value: index)
+        self.sharedKey.setInt64(4 + 4, value: collectionId.id)
+        return self.sharedKey
+    }
+    
+    private func lowerBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setInt32(0, value: namespace)
+        return key
+    }
+    
+    private func upperBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setInt32(0, value: namespace)
+        return key.successor
+    }
+    
+    
+    func getInfos(namespace: ItemCollectionId.Namespace) -> [(Int, ItemCollectionId, ItemCollectionInfo)] {
+        if let cachedInfo = self.cachedInfos[namespace] {
+            return cachedInfo
+        } else {
+            var infos: [(Int, ItemCollectionId, ItemCollectionInfo)] = []
+            self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in
+                if let info = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionInfo {
+                    infos.append((Int(key.getInt32(4)), ItemCollectionId(namespace: namespace, id: key.getInt64(4 + 4)), info))
+                }
+                return true
+            }, limit: 0)
+            self.cachedInfos[namespace] = infos
+            return infos
+        }
+    }
+    
+    func getIds(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId] {
+        if let cachedInfo = self.cachedInfos[namespace] {
+            return cachedInfo.map { $0.1 }
+        } else {
+            var ids: [ItemCollectionId] = []
+            self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in
+                ids.append(ItemCollectionId(namespace: namespace, id: key.getInt64(4 + 4)))
+                return true
+            }, limit: 0)
+            return ids
+        }
+    }
+    
+    func getIndex(id: ItemCollectionId) -> Int32? {
+        var index: Int32?
+        self.valueBox.range(self.table, start: self.lowerBound(namespace: id.namespace), end: self.upperBound(namespace: id.namespace), keys: { key in
+            let keyId = ItemCollectionId(namespace: id.namespace, id: key.getInt64(4 + 4))
+            if keyId == id {
+                index = key.getInt32(4)
+                return false
+            }
+            return true
+        }, limit: 0)
+        return index
+    }
+    
+    func getInfo(id: ItemCollectionId) -> ItemCollectionInfo? {
+        var infoKey: ValueBoxKey?
+        self.valueBox.range(self.table, start: self.lowerBound(namespace: id.namespace), end: self.upperBound(namespace: id.namespace), keys: { key in
+            let keyId = ItemCollectionId(namespace: id.namespace, id: key.getInt64(4 + 4))
+            if keyId == id {
+                infoKey = key
+                return false
+            }
+            return true
+        }, limit: 0)
+        if let infoKey = infoKey, let value = self.valueBox.get(self.table, key: infoKey), let info = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionInfo  {
+            return info
+        } else {
+            return nil
+        }
+    }
+    
+    func lowerCollectionId(namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, index: Int32) -> (ItemCollectionId, Int32)? {
+        var currentNamespace = collectionId.namespace
+        var currentKey = self.key(collectionId: collectionId, index: index)
+        while true {
+            var resultCollectionIdAndIndex: (ItemCollectionId, Int32)?
+            self.valueBox.range(self.table, start: currentKey, end: self.lowerBound(namespace: currentNamespace), keys: { key in
+                resultCollectionIdAndIndex = (ItemCollectionId(namespace: currentNamespace, id: key.getInt64(4 + 4)), key.getInt32(4))
+                return true
+            }, limit: 1)
+            if let resultCollectionIdAndIndex = resultCollectionIdAndIndex {
+                return resultCollectionIdAndIndex
+            } else {
+                let index = namespaceList.firstIndex(of: currentNamespace)!
+                if index == 0 {
+                    return nil
+                } else {
+                    currentNamespace = namespaceList[index - 1]
+                    currentKey = self.upperBound(namespace: currentNamespace)
+                }
+            }
+        }
+    }
+    
+    func higherCollectionId(namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, index: Int32) -> (ItemCollectionId, Int32)? {
+        var currentNamespace = collectionId.namespace
+        var currentKey = self.key(collectionId: collectionId, index: index)
+        while true {
+            var resultCollectionIdAndIndex: (ItemCollectionId, Int32)?
+            self.valueBox.range(self.table, start: currentKey, end: self.upperBound(namespace: currentNamespace), keys: { key in
+                resultCollectionIdAndIndex = (ItemCollectionId(namespace: currentNamespace, id: key.getInt64(4 + 4)), key.getInt32(4))
+                return true
+            }, limit: 1)
+            if let resultCollectionIdAndIndex = resultCollectionIdAndIndex {
+                return resultCollectionIdAndIndex
+            } else {
+                let index = namespaceList.firstIndex(of: currentNamespace)!
+                if index == namespaceList.count - 1 {
+                    return nil
+                } else {
+                    currentNamespace = namespaceList[index + 1]
+                    currentKey = self.lowerBound(namespace: currentNamespace)
+                }
+            }
+        }
+    }
+    
+    func replaceInfos(namespace: ItemCollectionId.Namespace, infos: [(ItemCollectionId, ItemCollectionInfo)]) {
+        self.cachedInfos.removeAll()
+        
+        var currentCollectionKeys: [ValueBoxKey] = []
+        self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), keys: { key in
+            currentCollectionKeys.append(key)
+            return true
+        }, limit: 0)
+        
+        for key in currentCollectionKeys {
+            self.valueBox.remove(self.table, key: key, secure: false)
+        }
+        
+        var index: Int32 = 0
+        let sharedEncoder = PostboxEncoder()
+        for (id, info) in infos {
+            sharedEncoder.reset()
+            sharedEncoder.encodeRootObject(info)
+            withExtendedLifetime(sharedEncoder, {
+                self.valueBox.set(self.table, key: self.key(collectionId: id, index: index), value: sharedEncoder.readBufferNoCopy())
+            })
+            index += 1
+        }
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedInfos.removeAll()
+    }
+    
+    override func beforeCommit() {
+    }
+}

submodules/Postbox/Sources/ItemCollectionInfoView.swift

@@ -0,0 +1,69 @@
+import Foundation
+
+final class MutableItemCollectionInfoView: MutablePostboxView {
+    let id: ItemCollectionId
+    var info: ItemCollectionInfo?
+    
+    init(postbox: PostboxImpl, id: ItemCollectionId) {
+        self.id = id
+        let infos = postbox.itemCollectionInfoTable.getInfos(namespace: id.namespace)
+        for (_, infoId, info) in infos {
+            if id == infoId {
+                self.info = info
+                break
+            }
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if transaction.currentItemCollectionInfosOperations.isEmpty {
+            return false
+        }
+        
+        var updated = false
+        
+        var reloadInfosNamespaces = Set<ItemCollectionId.Namespace>()
+        for operation in transaction.currentItemCollectionInfosOperations {
+            switch operation {
+                case let .replaceInfos(namespace):
+                    reloadInfosNamespaces.insert(namespace)
+            }
+        }
+        
+        if !reloadInfosNamespaces.isEmpty && reloadInfosNamespaces.contains(self.id.namespace) {
+            updated = true
+            
+            let infos = postbox.itemCollectionInfoTable.getInfos(namespace: id.namespace)
+            var found = false
+            for (_, infoId, info) in infos {
+                if id == infoId {
+                    self.info = info
+                    found = true
+                    break
+                }
+            }
+            if !found {
+                self.info = nil
+            }
+        }
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return ItemCollectionInfoView(self)
+    }
+}
+
+public final class ItemCollectionInfoView: PostboxView {
+    public let id: ItemCollectionId
+    public let info: ItemCollectionInfo?
+    
+    init(_ view: MutableItemCollectionInfoView) {
+        self.id = view.id
+        self.info = view.info
+    }
+}

submodules/Postbox/Sources/ItemCollectionInfosView.swift

@@ -0,0 +1,109 @@
+import Foundation
+
+public final class ItemCollectionInfoEntry {
+    public let id: ItemCollectionId
+    public let info: ItemCollectionInfo
+    public let count: Int32
+    public let firstItem: ItemCollectionItem?
+    
+    init(id: ItemCollectionId, info: ItemCollectionInfo, count: Int32, firstItem: ItemCollectionItem?) {
+        self.id = id
+        self.info = info
+        self.count = count
+        self.firstItem = firstItem
+    }
+}
+
+final class MutableItemCollectionInfosView: MutablePostboxView {
+    let namespaces: [ItemCollectionId.Namespace]
+    var entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]]
+    
+    init(postbox: PostboxImpl, namespaces: [ItemCollectionId.Namespace]) {
+        self.namespaces = namespaces
+        
+        var entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]] = [:]
+        for namespace in namespaces {
+            let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace)
+            var entries: [ItemCollectionInfoEntry] = []
+            for (_, id, info) in infos {
+                let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first
+                entries.append(ItemCollectionInfoEntry(id: id, info: info, count: postbox.itemCollectionItemTable.itemCount(collectionId: id), firstItem: firstItem))
+            }
+            entriesByNamespace[namespace] = entries
+        }
+        self.entriesByNamespace = entriesByNamespace
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        if transaction.currentItemCollectionInfosOperations.isEmpty && transaction.currentItemCollectionItemsOperations.isEmpty {
+            return false
+        }
+        
+        var updated = false
+        
+        var reloadInfosNamespaces = Set<ItemCollectionId.Namespace>()
+        var reloadTopItemCollectionIds = Set<ItemCollectionId>()
+        for operation in transaction.currentItemCollectionInfosOperations {
+            switch operation {
+                case let .replaceInfos(namespace):
+                    reloadInfosNamespaces.insert(namespace)
+            }
+        }
+        for (id, operations) in transaction.currentItemCollectionItemsOperations {
+            for operation in operations {
+                switch operation {
+                    case .replaceItems:
+                        reloadTopItemCollectionIds.insert(id)
+                }
+            }
+        }
+        if !reloadInfosNamespaces.isEmpty {
+            updated = true
+            
+            var entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]] = [:]
+            for namespace in self.namespaces {
+                let infos = postbox.itemCollectionInfoTable.getInfos(namespace: namespace)
+                var entries: [ItemCollectionInfoEntry] = []
+                for (_, id, info) in infos {
+                    let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first
+                    entries.append(ItemCollectionInfoEntry(id: id, info: info, count: postbox.itemCollectionItemTable.itemCount(collectionId: id), firstItem: firstItem))
+                }
+                entriesByNamespace[namespace] = entries
+            }
+            self.entriesByNamespace = entriesByNamespace
+        } else if !reloadTopItemCollectionIds.isEmpty {
+            var entriesByNamespace = self.entriesByNamespace
+            for (namespace, entries) in self.entriesByNamespace {
+                var items: [ItemCollectionInfoEntry] = []
+                for i in 0 ..< entries.count {
+                    if reloadTopItemCollectionIds.contains(entries[i].id) {
+                        updated = true
+                        let firstItem = postbox.itemCollectionItemTable.higherItems(collectionId: entries[i].id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first
+                        items.append(ItemCollectionInfoEntry(id: entries[i].id, info: entries[i].info, count: postbox.itemCollectionItemTable.itemCount(collectionId: entries[i].id), firstItem: firstItem))
+                    } else {
+                        items.append(entriesByNamespace[namespace]![i])
+                    }
+                }
+                entriesByNamespace[namespace] = items
+            }
+            self.entriesByNamespace = entriesByNamespace
+        }
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return ItemCollectionInfosView(self)
+    }
+}
+
+public final class ItemCollectionInfosView: PostboxView {
+    public let entriesByNamespace: [ItemCollectionId.Namespace: [ItemCollectionInfoEntry]]
+    
+    init(_ view: MutableItemCollectionInfosView) {
+        self.entriesByNamespace = view.entriesByNamespace
+    }
+}

submodules/Postbox/Sources/ItemCollectionItemTable.swift

@@ -0,0 +1,261 @@
+import Foundation
+
+enum ItemCollectionItemsOperation {
+    case replaceItems
+}
+
+struct ItemCollectionItemReverseIndexReference: Equatable, Hashable, ReverseIndexReference {
+    let collectionId: ItemCollectionId
+    let itemIndex: ItemCollectionItemIndex
+    
+    static func <(lhs: ItemCollectionItemReverseIndexReference, rhs: ItemCollectionItemReverseIndexReference) -> Bool {
+        if lhs.collectionId != rhs.collectionId {
+            return lhs.collectionId < rhs.collectionId
+        } else {
+            return lhs.itemIndex < rhs.itemIndex
+        }
+    }
+    
+    static func decodeArray(_ buffer: MemoryBuffer) -> [ItemCollectionItemReverseIndexReference] {
+        assert(buffer.length % (4 + 8 + 4 + 8) == 0)
+        var references: [ItemCollectionItemReverseIndexReference] = []
+        references.reserveCapacity(buffer.length % (4 + 8 + 4 + 8))
+        withExtendedLifetime(buffer, {
+            let readBuffer = ReadBuffer(memoryBufferNoCopy: buffer)
+            for _ in 0 ..< buffer.length / (4 + 8 + 4 + 8) {
+                var collectionIdNamespace: Int32 = 0
+                var collectionIdId: Int64 = 0
+                var indexIdIndex: Int32 = 0
+                var indexIdId: Int64 = 0
+                readBuffer.read(&collectionIdNamespace, offset: 0, length: 4)
+                readBuffer.read(&collectionIdId, offset: 0, length: 8)
+                readBuffer.read(&indexIdIndex, offset: 0, length: 4)
+                readBuffer.read(&indexIdId, offset: 0, length: 8)
+                references.append(ItemCollectionItemReverseIndexReference(collectionId: ItemCollectionId(namespace: collectionIdNamespace, id: collectionIdId), itemIndex: ItemCollectionItemIndex(index: indexIdIndex, id: indexIdId)))
+            }
+        })
+        return references
+    }
+    
+    static func encodeArray(_ array: [ItemCollectionItemReverseIndexReference]) -> MemoryBuffer {
+        let buffer = WriteBuffer()
+        for reference in array {
+            var collectionIdNamespace: Int32 = reference.collectionId.namespace
+            var collectionIdId: Int64 = reference.collectionId.id
+            var indexIdIndex: Int32 = reference.itemIndex.index
+            var indexIdId: Int64 = reference.itemIndex.id
+            
+            buffer.write(&collectionIdNamespace, offset: 0, length: 4)
+            buffer.write(&collectionIdId, offset: 0, length: 8)
+            buffer.write(&indexIdIndex, offset: 0, length: 4)
+            buffer.write(&indexIdId, offset: 0, length: 8)
+        }
+        return buffer
+    }
+}
+
+final class ItemCollectionItemTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: false)
+    }
+    
+    private let reverseIndexTable: ReverseIndexReferenceTable<ItemCollectionItemReverseIndexReference>
+    
+    private let sharedKey = ValueBoxKey(length: 4 + 8 + 4 + 8)
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, reverseIndexTable: ReverseIndexReferenceTable<ItemCollectionItemReverseIndexReference>) {
+        self.reverseIndexTable = reverseIndexTable
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(collectionId: ItemCollectionId, index: ItemCollectionItemIndex) -> ValueBoxKey {
+        self.sharedKey.setInt32(0, value: collectionId.namespace)
+        self.sharedKey.setInt64(4, value: collectionId.id)
+        self.sharedKey.setInt32(4 + 8, value: index.index)
+        self.sharedKey.setInt64(4 + 8 + 4, value: index.id)
+        return self.sharedKey
+    }
+    
+    private func lowerBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setInt32(0, value: namespace)
+        return key
+    }
+    
+    private func upperBound(namespace: ItemCollectionId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4)
+        key.setInt32(0, value: namespace)
+        return key.successor
+    }
+    
+    private func lowerBound(collectionId: ItemCollectionId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4 + 8)
+        key.setInt32(0, value: collectionId.namespace)
+        key.setInt64(4, value: collectionId.id)
+        return key
+    }
+    
+    private func upperBound(collectionId: ItemCollectionId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4 + 8)
+        key.setInt32(0, value: collectionId.namespace)
+        key.setInt64(4, value: collectionId.id)
+        return key.successor
+    }
+    
+    func lowerItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] {
+        var items: [ItemCollectionItem] = []
+        self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.lowerBound(collectionId: collectionId), values: { _, value in
+            if let item = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionItem {
+                items.append(item)
+            } else {
+                assertionFailure()
+            }
+            return true
+        }, limit: count)
+        return items
+    }
+    
+    func itemCount(collectionId: ItemCollectionId) -> Int32 {
+        var count: Int32 = 0
+        self.valueBox.range(self.table, start: self.upperBound(collectionId: collectionId), end: self.lowerBound(collectionId: collectionId), keys: { key in
+            let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4))
+            count = itemIndex.index + 1
+            return true
+        }, limit: 1)
+        return count
+    }
+    
+    func higherItems(collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] {
+        var items: [ItemCollectionItem] = []
+        self.valueBox.range(self.table, start: self.key(collectionId: collectionId, index: itemIndex), end: self.upperBound(collectionId: collectionId), values: { _, value in
+            if let item = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionItem {
+                items.append(item)
+            } else {
+                assertionFailure()
+            }
+            return true
+        }, limit: count)
+        return items
+    }
+    
+    func getItems(namespace: ItemCollectionId.Namespace) -> [ItemCollectionId: [ItemCollectionItem]] {
+        var items: [ItemCollectionId: [ItemCollectionItem]] = [:]
+        self.valueBox.range(self.table, start: self.lowerBound(namespace: namespace), end: self.upperBound(namespace: namespace), values: { key, value in
+            let collectionId = ItemCollectionId(namespace: namespace, id: key.getInt64(4))
+            //let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4))
+            if let item = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionItem {
+                if items[collectionId] != nil {
+                    items[collectionId]!.append(item)
+                } else {
+                    items[collectionId] = [item]
+                }
+            } else {
+                assertionFailure()
+            }
+            return true
+        }, limit: 0)
+        return items
+    }
+    
+    func collectionItems(collectionId: ItemCollectionId) -> [ItemCollectionItem] {
+        var items: [ItemCollectionItem] = []
+        self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), values: { key, value in
+            //let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4))
+            if let item = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionItem {
+                items.append(item)
+            } else {
+                assertionFailure()
+            }
+            return true
+        }, limit: 0)
+        return items
+    }
+    
+    func replaceItems(collectionId: ItemCollectionId, items: [ItemCollectionItem]) {
+        let updatedIndices = Set(items.map({ $0.index }))
+        var itemByIndex: [ItemCollectionItemIndex: ItemCollectionItem] = [:]
+        for item in items {
+            itemByIndex[item.index] = item
+        }
+        
+        var currentIndices = Set<ItemCollectionItemIndex>()
+        
+        var removedIndexKeys: [ItemCollectionItemIndex: [MemoryBuffer]] = [:]
+        
+        self.valueBox.range(self.table, start: self.lowerBound(collectionId: collectionId), end: self.upperBound(collectionId: collectionId), values: { key, value in
+            let itemIndex = ItemCollectionItemIndex(index: key.getInt32(4 + 8), id: key.getInt64(4 + 8 + 4))
+            currentIndices.insert(itemIndex)
+            
+            if !updatedIndices.contains(itemIndex) {
+                if let item = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionItem {
+                    if !item.indexKeys.isEmpty {
+                        removedIndexKeys[itemIndex] = item.indexKeys
+                    }
+                } else {
+                    assertionFailure()
+                }
+            }
+            return true
+        }, limit: 0)
+        
+        let addedIndices = updatedIndices.subtracting(currentIndices)
+        let removedIndices = currentIndices.subtracting(updatedIndices)
+        
+        for index in removedIndices {
+            self.valueBox.remove(self.table, key: self.key(collectionId: collectionId, index: index), secure: false)
+            if let indexKeys = removedIndexKeys[index] {
+                self.reverseIndexTable.remove(namespace: ReverseIndexNamespace(collectionId.namespace), reference: ItemCollectionItemReverseIndexReference(collectionId: collectionId, itemIndex: index), tokens: indexKeys.map({ ValueBoxKey($0) }))
+            }
+        }
+        
+        let sharedEncoder = PostboxEncoder()
+        for index in addedIndices {
+            let item = itemByIndex[index]!
+            sharedEncoder.reset()
+            sharedEncoder.encodeRootObject(item)
+            withExtendedLifetime(sharedEncoder, {
+                self.valueBox.set(self.table, key: self.key(collectionId: collectionId, index: index), value: sharedEncoder.readBufferNoCopy())
+            })
+            if !item.indexKeys.isEmpty {
+                self.reverseIndexTable.add(namespace: ReverseIndexNamespace(collectionId.namespace), reference: ItemCollectionItemReverseIndexReference(collectionId: collectionId, itemIndex: index), tokens: item.indexKeys.map({ ValueBoxKey($0) }))
+            }
+        }
+    }
+    
+    func searchIndexedItems(namespace: ItemCollectionId.Namespace, query: ItemCollectionSearchQuery) -> [ItemCollectionId: [ItemCollectionItem]] {
+        let references: [ItemCollectionItemReverseIndexReference]
+        switch query {
+            case let .exact(token):
+                references = self.reverseIndexTable.exactReferences(namespace: ReverseIndexNamespace(namespace), token: token)
+            case let .matching(tokens):
+                references = Array(self.reverseIndexTable.matchingReferences(namespace: ReverseIndexNamespace(namespace), tokens: tokens))
+            case let .any(tokens):
+                references = Array(self.reverseIndexTable.matchingReferences(namespace: ReverseIndexNamespace(namespace), tokens: tokens, union: true))
+        }
+        var resultsByCollectionId: [ItemCollectionId: [(ItemCollectionItemIndex, ItemCollectionItem)]] = [:]
+        for reference in references {
+            if let value = self.valueBox.get(self.table, key: self.key(collectionId: reference.collectionId, index: reference.itemIndex)), let item = PostboxDecoder(buffer: value).decodeRootObject() as? ItemCollectionItem {
+                if resultsByCollectionId[reference.collectionId] == nil {
+                    resultsByCollectionId[reference.collectionId] = [(reference.itemIndex, item)]
+                } else {
+                    resultsByCollectionId[reference.collectionId]!.append((reference.itemIndex, item))
+                }
+            } else {
+                assertionFailure()
+            }
+        }
+        var results: [ItemCollectionId: [ItemCollectionItem]] = [:]
+        for collectionId in resultsByCollectionId.keys {
+            results[collectionId] = resultsByCollectionId[collectionId]!.sorted(by: { $0.0 < $1.0 }).map({ $0.1 })
+        }
+        return results
+    }
+
+    override func clearMemoryCache() {
+        
+    }
+    
+    override func beforeCommit() {
+        
+    }
+}

submodules/Postbox/Sources/ItemCollectionsView.swift

@@ -0,0 +1,309 @@
+import Foundation
+
+public struct ItemCollectionViewEntryIndex: Comparable {
+    public let collectionIndex: Int32
+    public let collectionId: ItemCollectionId
+    public let itemIndex: ItemCollectionItemIndex
+    
+    public init(collectionIndex: Int32, collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex) {
+        self.collectionIndex = collectionIndex
+        self.collectionId = collectionId
+        self.itemIndex = itemIndex
+    }
+    
+    public static func ==(lhs: ItemCollectionViewEntryIndex, rhs: ItemCollectionViewEntryIndex) -> Bool {
+        return lhs.collectionIndex == rhs.collectionIndex && lhs.collectionId == rhs.collectionId && lhs.itemIndex == rhs.itemIndex
+    }
+    
+    public static func <(lhs: ItemCollectionViewEntryIndex, rhs: ItemCollectionViewEntryIndex) -> Bool {
+        if lhs.collectionIndex == rhs.collectionIndex {
+            if lhs.itemIndex == rhs.itemIndex {
+                return lhs.collectionId < rhs.collectionId
+            } else {
+                return lhs.itemIndex < rhs.itemIndex
+            }
+        } else {
+            return lhs.collectionIndex < rhs.collectionIndex
+        }
+    }
+    
+    public static func lowerBound(collectionIndex: Int32, collectionId: ItemCollectionId) -> ItemCollectionViewEntryIndex {
+        return ItemCollectionViewEntryIndex(collectionIndex: collectionIndex, collectionId: collectionId, itemIndex: ItemCollectionItemIndex(index: 0, id: 0))
+    }
+}
+
+public struct ItemCollectionViewEntry {
+    public let index: ItemCollectionViewEntryIndex
+    public let item: ItemCollectionItem
+    
+    public init(index: ItemCollectionViewEntryIndex, item: ItemCollectionItem) {
+        self.index = index
+        self.item = item
+    }
+}
+
+private func fetchLowerEntries(namespaces: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32, itemIndex: ItemCollectionItemIndex, count: Int, lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, lowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) -> [ItemCollectionViewEntry] {
+    
+    var entries: [ItemCollectionViewEntry] = []
+    
+    var currentCollectionIndex = collectionIndex
+    var currentCollectionId = collectionId
+    var currentItemIndex = itemIndex
+    
+    while true {
+        let remainingCount = count - entries.count
+        assert(remainingCount > 0)
+        let collectionItems = lowerItems(currentCollectionId, currentItemIndex, remainingCount)
+        for item in collectionItems {
+            entries.append(ItemCollectionViewEntry(index: ItemCollectionViewEntryIndex(collectionIndex: currentCollectionIndex, collectionId: currentCollectionId, itemIndex: item.index), item: item))
+        }
+        if entries.count >= count {
+            break
+        } else {
+            assert(collectionItems.count < remainingCount)
+            
+            if let (previousCollectionId, previousCollectionIndex) = lowerCollectionId(namespaces, currentCollectionId, currentCollectionIndex) {
+                currentCollectionIndex = previousCollectionIndex
+                currentCollectionId = previousCollectionId
+                currentItemIndex = ItemCollectionItemIndex.upperBound
+            } else {
+                break
+            }
+        }
+    }
+    
+    return entries
+}
+
+private func fetchHigherEntries(namespaces: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32, itemIndex: ItemCollectionItemIndex, count: Int, higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?, higherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) -> [ItemCollectionViewEntry] {
+    
+    var entries: [ItemCollectionViewEntry] = []
+    
+    var currentCollectionIndex = collectionIndex
+    var currentCollectionId = collectionId
+    var currentItemIndex = itemIndex
+    
+    while true {
+        let remainingCount = count - entries.count
+        assert(remainingCount > 0)
+        let collectionItems = higherItems(currentCollectionId, currentItemIndex, remainingCount)
+        for item in collectionItems {
+            entries.append(ItemCollectionViewEntry(index: ItemCollectionViewEntryIndex(collectionIndex: currentCollectionIndex, collectionId: currentCollectionId, itemIndex: item.index), item: item))
+        }
+        if entries.count >= count {
+            break
+        } else {
+            assert(collectionItems.count < remainingCount)
+            
+            if let (nextCollectionId, nextCollectionIndex) = higherCollectionId(namespaces, currentCollectionId, currentCollectionIndex) {
+                currentCollectionIndex = nextCollectionIndex
+                currentCollectionId = nextCollectionId
+                currentItemIndex = ItemCollectionItemIndex.lowerBound
+            } else {
+                break
+            }
+        }
+    }
+    
+    return entries
+}
+
+private func aroundEntries(namespaces: [ItemCollectionId.Namespace],
+                           aroundIndex: ItemCollectionViewEntryIndex?,
+                           count: Int,
+                           collectionIndexById: (ItemCollectionId) -> Int32?,
+                           lowerCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?,
+                           fetchLowerItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem],
+                           higherCollectionId: (_ namespaceList: [ItemCollectionId.Namespace], _ collectionId: ItemCollectionId, _ collectionIndex: Int32) -> (ItemCollectionId, Int32)?,
+                           fetchHigherItems: (_ collectionId: ItemCollectionId, _ itemIndex: ItemCollectionItemIndex, _ count: Int) -> [ItemCollectionItem]) -> ([ItemCollectionViewEntry], ItemCollectionViewEntry?, ItemCollectionViewEntry?) {
+    var lowerEntries: [ItemCollectionViewEntry] = []
+    var upperEntries: [ItemCollectionViewEntry] = []
+    var lower: ItemCollectionViewEntry?
+    var upper: ItemCollectionViewEntry?
+    
+    let selectedAroundIndex: ItemCollectionViewEntryIndex
+    if let aroundIndex = aroundIndex, let aroundCollectionIndex = collectionIndexById(aroundIndex.collectionId) {
+        selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: aroundCollectionIndex, collectionId: aroundIndex.collectionId, itemIndex: aroundIndex.itemIndex)
+    } else {
+        selectedAroundIndex = ItemCollectionViewEntryIndex(collectionIndex: 0, collectionId: ItemCollectionId(namespace: namespaces[0], id: 0), itemIndex: ItemCollectionItemIndex.lowerBound)
+    }
+    
+    let collectionId: ItemCollectionId = selectedAroundIndex.collectionId
+    let collectionIndex: Int32 = selectedAroundIndex.collectionIndex
+    let itemIndex: ItemCollectionItemIndex = selectedAroundIndex.itemIndex
+    
+    lowerEntries.append(contentsOf: fetchLowerEntries(namespaces: namespaces, collectionId: collectionId, collectionIndex: collectionIndex, itemIndex: itemIndex, count: count / 2 + 1, lowerCollectionId: lowerCollectionId, lowerItems: fetchLowerItems))
+    
+    let lowerIndices = lowerEntries.map { $0.index }
+    assert(lowerIndices.sorted() == lowerIndices.reversed())
+    
+    if lowerEntries.count >= count / 2 + 1 {
+        lower = lowerEntries.last
+        lowerEntries.removeLast()
+    }
+    
+    upperEntries.append(contentsOf: fetchHigherEntries(namespaces: namespaces, collectionId: collectionId, collectionIndex: collectionIndex, itemIndex: ItemCollectionItemIndex(index: itemIndex.index, id: max(0, itemIndex.id - 1)), count: count - lowerEntries.count + 1, higherCollectionId: higherCollectionId, higherItems: fetchHigherItems))
+    
+    let upperIndices = upperEntries.map { $0.index }
+    assert(upperIndices.sorted() == upperIndices)
+    
+    if upperEntries.count >= count - lowerEntries.count + 1 {
+        upper = upperEntries.last
+        upperEntries.removeLast()
+    }
+    
+    if lowerEntries.count != 0 && lowerEntries.count + upperEntries.count < count {
+        var additionalLowerEntries: [ItemCollectionViewEntry] = fetchLowerEntries(namespaces: namespaces, collectionId: lowerEntries.last!.index.collectionId, collectionIndex: lowerEntries.last!.index.collectionIndex, itemIndex: lowerEntries.last!.index.itemIndex, count: count - lowerEntries.count - upperEntries.count + 1, lowerCollectionId: lowerCollectionId, lowerItems: fetchLowerItems)
+        
+        if additionalLowerEntries.count >= count - lowerEntries.count + upperEntries.count + 1 {
+            lower = additionalLowerEntries.last
+            additionalLowerEntries.removeLast()
+        }
+        lowerEntries.append(contentsOf: additionalLowerEntries)
+    }
+    
+    var entries: [ItemCollectionViewEntry] = []
+    entries.append(contentsOf: lowerEntries.reversed())
+    entries.append(contentsOf: upperEntries)
+    return (entries: entries, lower: lower, upper: upper)
+}
+
+final class MutableItemCollectionsView {
+    let orderedItemListsViews: [MutableOrderedItemListView]
+    let namespaces: [ItemCollectionId.Namespace]
+    let requestedAroundIndex: ItemCollectionViewEntryIndex?
+    let requestedCount: Int
+    
+    var collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)]
+    var entries: [ItemCollectionViewEntry]
+    var lower: ItemCollectionViewEntry?
+    var higher: ItemCollectionViewEntry?
+    
+    init(postbox: PostboxImpl, orderedItemListsViews: [MutableOrderedItemListView], namespaces: [ItemCollectionId.Namespace], aroundIndex: ItemCollectionViewEntryIndex?, count: Int) {
+        self.orderedItemListsViews = orderedItemListsViews
+        self.namespaces = namespaces
+        self.requestedAroundIndex = aroundIndex
+        self.requestedCount = count
+        
+        self.collectionInfos = []
+        self.entries = []
+        self.lower = nil
+        self.higher = nil
+        
+        self.reload(postbox: postbox, aroundIndex: aroundIndex, count: count)
+    }
+    
+    private func lowerItems(postbox: PostboxImpl, collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] {
+        return postbox.itemCollectionItemTable.lowerItems(collectionId: collectionId, itemIndex: itemIndex, count: count)
+    }
+    
+    private func higherItems(postbox: PostboxImpl, collectionId: ItemCollectionId, itemIndex: ItemCollectionItemIndex, count: Int) -> [ItemCollectionItem] {
+        return postbox.itemCollectionItemTable.higherItems(collectionId: collectionId, itemIndex: itemIndex, count: count)
+    }
+    
+    private func lowerCollectionId(postbox: PostboxImpl, namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32) -> (ItemCollectionId, Int32)? {
+        return postbox.itemCollectionInfoTable.lowerCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex)
+    }
+    
+    private func higherCollectionId(postbox: PostboxImpl, namespaceList: [ItemCollectionId.Namespace], collectionId: ItemCollectionId, collectionIndex: Int32) -> (ItemCollectionId, Int32)? {
+        return postbox.itemCollectionInfoTable.higherCollectionId(namespaceList: namespaceList, collectionId: collectionId, index: collectionIndex)
+    }
+    
+    private func reload(postbox: PostboxImpl, aroundIndex: ItemCollectionViewEntryIndex?, count: Int) {
+        self.collectionInfos = []
+        for namespace in namespaces {
+            for (_, id, info) in postbox.itemCollectionInfoTable.getInfos(namespace: namespace) {
+                let item = self.higherItems(postbox: postbox, collectionId: id, itemIndex: ItemCollectionItemIndex.lowerBound, count: 1).first
+                self.collectionInfos.append((id, info, item))
+            }
+        }
+        
+        let (entries, lower, higher) = aroundEntries(
+            namespaces: namespaces,
+            aroundIndex: aroundIndex,
+            count: count, collectionIndexById: { id in
+                return postbox.itemCollectionInfoTable.getIndex(id: id)
+            },
+            lowerCollectionId: { namespaceList, collectionId, collectionIndex in
+                return self.lowerCollectionId(postbox: postbox, namespaceList: namespaceList, collectionId: collectionId, collectionIndex: collectionIndex)
+            },
+            fetchLowerItems: { collectionId, itemIndex, count in
+                return self.lowerItems(postbox: postbox, collectionId: collectionId, itemIndex: itemIndex, count: count)
+            },
+            higherCollectionId: { namespaceList, collectionId, collectionIndex in
+                return self.higherCollectionId(postbox: postbox, namespaceList: namespaceList, collectionId: collectionId, collectionIndex: collectionIndex)
+            },
+            fetchHigherItems: {
+                collectionId, itemIndex, count in
+                return self.higherItems(postbox: postbox, collectionId: collectionId, itemIndex: itemIndex, count: count)
+            }
+        )
+        
+        self.entries = entries
+        self.lower = lower
+        self.higher = higher
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if !transaction.currentOrderedItemListOperations.isEmpty {
+            for view in self.orderedItemListsViews {
+                if view.replay(postbox: postbox, transaction: transaction) {
+                    updated = true
+                }
+            }
+        }
+        
+        
+        var reloadNamespaces = Set<ItemCollectionId.Namespace>()
+        for operation in transaction.currentItemCollectionInfosOperations {
+            switch operation {
+                case let .replaceInfos(namespace):
+                    reloadNamespaces.insert(namespace)
+            }
+        }
+        
+        for (id, operations) in transaction.currentItemCollectionItemsOperations {
+            for operation in operations {
+                switch operation {
+                    case .replaceItems:
+                        reloadNamespaces.insert(id.namespace)
+                }
+            }
+        }
+        
+        var shouldReloadEntries = false
+        if !reloadNamespaces.isEmpty {
+            for namespace in self.namespaces {
+                if reloadNamespaces.contains(namespace) {
+                    shouldReloadEntries = true
+                    break
+                }
+            }
+        }
+        
+        if shouldReloadEntries {
+            self.reload(postbox: postbox, aroundIndex: self.requestedAroundIndex, count: self.requestedCount)
+            updated = true
+        }
+        
+        return updated
+    }
+}
+
+public final class ItemCollectionsView {
+    public let orderedItemListsViews: [OrderedItemListView]
+    public let collectionInfos: [(ItemCollectionId, ItemCollectionInfo, ItemCollectionItem?)]
+    public let entries: [ItemCollectionViewEntry]
+    public let lower: ItemCollectionViewEntry?
+    public let higher: ItemCollectionViewEntry?
+    
+    init(_ mutableView: MutableItemCollectionsView) {
+        self.orderedItemListsViews = mutableView.orderedItemListsViews.map { OrderedItemListView($0) }
+        self.collectionInfos = mutableView.collectionInfos
+        self.entries = mutableView.entries
+        self.lower = mutableView.lower
+        self.higher = mutableView.higher
+    }
+}

submodules/Postbox/Sources/KeychainTable.swift

@@ -0,0 +1,26 @@
+import Foundation
+
+final class KeychainTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: false)
+    }
+    
+    private func key(_ string: String) -> ValueBoxKey {
+        return ValueBoxKey(string)
+    }
+    
+    func get(_ key: String) -> Data? {
+        if let value = self.valueBox.get(self.table, key: self.key(key)) {
+            return Data(bytes: value.memory.assumingMemoryBound(to: UInt8.self), count: value.length)
+        }
+        return nil
+    }
+    
+    func set(_ key: String, value: Data) {
+        self.valueBox.set(self.table, key: self.key(key), value: MemoryBuffer(data: value))
+    }
+    
+    func remove(_ key: String) {
+        self.valueBox.remove(self.table, key: self.key(key), secure: false)
+    }
+}

submodules/Postbox/Sources/LocalMessageHistoryTagsTable.swift

@@ -0,0 +1,64 @@
+import Foundation
+
+enum IntermediateMessageHistoryLocalTagsOperation {
+    case Insert(LocalMessageTags, MessageId)
+    case Remove(LocalMessageTags, MessageId)
+    case Update(LocalMessageTags, MessageId)
+}
+
+final class LocalMessageHistoryTagsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 4 + 4 + 4 + 8)
+    
+    private func key(_ id: MessageId, tag: LocalMessageTags) -> ValueBoxKey {
+        assert(tag.isSingleTag)
+        self.sharedKey.setUInt32(0, value: tag.rawValue)
+        self.sharedKey.setInt32(4, value: id.namespace)
+        self.sharedKey.setInt32(4 + 4, value: id.id)
+        self.sharedKey.setInt64(4 + 4 + 4, value: id.peerId.toInt64())
+        
+        return self.sharedKey
+    }
+    
+    private func lowerBound(tag: LocalMessageTags) -> ValueBoxKey {
+        assert(tag.isSingleTag)
+        
+        let key = ValueBoxKey(length: 4)
+        key.setUInt32(0, value: tag.rawValue)
+        return key
+    }
+    
+    private func upperBound(tag: LocalMessageTags) -> ValueBoxKey {
+        assert(tag.isSingleTag)
+        
+        let key = ValueBoxKey(length: 4)
+        key.setUInt32(0, value: tag.rawValue)
+        return key.successor
+    }
+    
+    func set(id: MessageId, tags: LocalMessageTags, previousTags: LocalMessageTags, operations: inout [IntermediateMessageHistoryLocalTagsOperation]) {
+        let addedTags = tags.subtracting(previousTags)
+        let removedTags = previousTags.subtracting(tags)
+        for tag in addedTags {
+            self.valueBox.set(self.table, key: self.key(id, tag: tag), value: MemoryBuffer())
+            operations.append(.Insert(tag, id))
+        }
+        for tag in removedTags {
+            self.valueBox.remove(self.table, key: self.key(id, tag: tag), secure: false)
+            operations.append(.Remove(tag, id))
+        }
+    }
+    
+    func get(tag: LocalMessageTags) -> [MessageId] {
+        assert(tag.isSingleTag)
+        var ids: [MessageId] = []
+        self.valueBox.range(self.table, start: self.lowerBound(tag: tag), end: self.upperBound(tag: tag), keys: { key in
+            ids.append(MessageId(peerId: PeerId(key.getInt64(4 + 4 + 4)), namespace: key.getInt32(4), id: key.getInt32(4 + 4)))
+            return true
+        }, limit: 0)
+        return ids
+    }
+}

submodules/Postbox/Sources/LocalMessageTagsView.swift

@@ -0,0 +1,71 @@
+import Foundation
+
+final class MutableLocalMessageTagsView: MutablePostboxView {
+    private let tag: LocalMessageTags
+    fileprivate var messages: [MessageId: Message] = [:]
+    
+    init(postbox: PostboxImpl, tag: LocalMessageTags) {
+        self.tag = tag
+        for id in postbox.localMessageHistoryTagsTable.get(tag: tag) {
+            if let message = postbox.getMessage(id) {
+                self.messages[message.id] = message
+            } else {
+                //assertionFailure()
+            }
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        for operation in transaction.currentLocalTagsOperations {
+            switch operation {
+                case let .Insert(tag, id):
+                    if tag == self.tag {
+                        if let message = postbox.getMessage(id) {
+                            self.messages[id] = message
+                            updated = true
+                        } else {
+                            assertionFailure()
+                        }
+                    }
+                case let .Remove(tag, id):
+                    if tag == self.tag {
+                        if self.messages[id] != nil {
+                            self.messages.removeValue(forKey: id)
+                            updated = true
+                        }
+                    }
+                case let .Update(tag, id):
+                    if tag == self.tag {
+                        if self.messages[id] != nil {
+                            if let message = postbox.getMessage(id) {
+                                self.messages[id] = message
+                                updated = true
+                            } else {
+                                assertionFailure()
+                            }
+                        }
+                    }
+            }
+        }
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return LocalMessageTagsView(self)
+    }
+}
+
+public final class LocalMessageTagsView: PostboxView {
+    public var messages: [MessageId: Message] = [:]
+    
+    init(_ view: MutableLocalMessageTagsView) {
+        self.messages = view.messages
+    }
+}
+
+

submodules/Postbox/Sources/MappedFile.swift

@@ -0,0 +1,51 @@
+import Foundation
+
+public final class MappedFile {
+    private var handle: Int32
+    private var currentSize: Int
+    private var memory: UnsafeMutableRawPointer
+    
+    public init(path: String) {
+        self.handle = open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR)
+        
+        var value = stat()
+        stat(path, &value)
+        self.currentSize = Int(value.st_size)
+        
+        self.memory = mmap(nil, self.currentSize, PROT_READ | PROT_WRITE, MAP_SHARED, self.handle, 0)
+    }
+    
+    deinit {
+        munmap(self.memory, self.currentSize)
+        close(self.handle)
+    }
+    
+    public var size: Int {
+        get {
+            return self.currentSize
+        } set(value) {
+            if value != self.currentSize {
+                munmap(self.memory, self.currentSize)
+                ftruncate(self.handle, off_t(value))
+                self.currentSize = value
+                self.memory = mmap(nil, self.currentSize, PROT_READ | PROT_WRITE, MAP_SHARED, self.handle, 0)
+            }
+        }
+    }
+    
+    public func synchronize() {
+        msync(self.memory, self.currentSize, MS_ASYNC)
+    }
+    
+    public func write(at range: Range<Int>, from data: UnsafeRawPointer) {
+        memcpy(self.memory.advanced(by: range.lowerBound), data, range.count)
+    }
+    
+    public func read(at range: Range<Int>, to data: UnsafeMutableRawPointer) {
+        memcpy(data, self.memory.advanced(by: range.lowerBound), range.count)
+    }
+    
+    public func clear() {
+        memset(self.memory, 0, self.currentSize)
+    }
+}

submodules/Postbox/Sources/Media.swift

@@ -0,0 +1,152 @@
+import Foundation
+
+public struct MediaId: Hashable, Comparable, PostboxCoding, CustomStringConvertible, Codable {
+    public typealias Namespace = Int32
+    public typealias Id = Int64
+    
+    public let namespace: Namespace
+    public let id: Id
+    
+    public var description: String {
+        get {
+            return "\(namespace):\(id)"
+        }
+    }
+    
+    public init(namespace: Namespace, id: Id) {
+        self.namespace = namespace
+        self.id = id
+    }
+    
+    public init(_ buffer: ReadBuffer) {
+        var namespace: Int32 = 0
+        var id: Int64 = 0
+        
+        memcpy(&namespace, buffer.memory + buffer.offset, 4)
+        self.namespace = namespace
+        memcpy(&id, buffer.memory + (buffer.offset + 4), 8)
+        self.id = id
+        buffer.offset += 12
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.namespace = decoder.decodeInt32ForKey("n", orElse: 0)
+        self.id = decoder.decodeInt64ForKey("i", orElse: 0)
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeInt32(self.namespace, forKey: "n")
+        encoder.encodeInt64(self.id, forKey: "i")
+    }
+    
+    public static func <(lhs: MediaId, rhs: MediaId) -> Bool {
+        if lhs.namespace != rhs.namespace {
+            return lhs.namespace < rhs.namespace
+        } else {
+            return lhs.id < rhs.id
+        }
+    }
+    
+    public func encodeToBuffer(_ buffer: WriteBuffer) {
+        var namespace = self.namespace
+        var id = self.id
+        buffer.write(&namespace, offset: 0, length: 4);
+        buffer.write(&id, offset: 0, length: 8);
+    }
+    
+    public static func encodeArrayToBuffer(_ array: [MediaId], buffer: WriteBuffer) {
+        var length: Int32 = Int32(array.count)
+        buffer.write(&length, offset: 0, length: 4)
+        for id in array {
+            id.encodeToBuffer(buffer)
+        }
+    }
+    
+    public static func decodeArrayFromBuffer(_ buffer: ReadBuffer) -> [MediaId] {
+        var length: Int32 = 0
+        memcpy(&length, buffer.memory, 4)
+        buffer.offset += 4
+        var i = 0
+        var array: [MediaId] = []
+        while i < Int(length) {
+            array.append(MediaId(buffer))
+            i += 1
+        }
+        return array
+    }
+}
+
+public protocol Media: AnyObject, PostboxCoding {
+    var id: MediaId? { get }
+    var peerIds: [PeerId] { get }
+    var storyIds: [StoryId] { get }
+    
+    var indexableText: String? { get }
+    
+    func isLikelyToBeUpdated() -> Bool
+
+    func preventsAutomaticMessageSendingFailure() -> Bool
+    
+    func isEqual(to other: Media) -> Bool
+    func isSemanticallyEqual(to other: Media) -> Bool
+}
+
+public extension Media {
+    var storyIds: [StoryId] {
+        return []
+    }
+}
+
+public func areMediaArraysEqual(_ lhs: [Media], _ rhs: [Media]) -> Bool {
+    if lhs.count != rhs.count {
+        return false
+    }
+    for i in 0 ..< lhs.count {
+        if !lhs[i].isEqual(to: rhs[i]) {
+            return false
+        }
+    }
+    return true
+}
+
+public func areMediaArraysSemanticallyEqual(_ lhs: [Media], _ rhs: [Media]) -> Bool {
+    if lhs.count != rhs.count {
+        return false
+    }
+    for i in 0 ..< lhs.count {
+        if !lhs[i].isSemanticallyEqual(to: rhs[i]) {
+            return false
+        }
+    }
+    return true
+}
+
+public func areMediaDictionariesEqual(_ lhs: [MediaId: Media], _ rhs: [MediaId: Media]) -> Bool {
+    if lhs.count != rhs.count {
+        return false
+    }
+    for (key, value) in lhs {
+        if let rhsValue = rhs[key] {
+            if !value.isEqual(to: rhsValue) {
+                return false
+            }
+        } else {
+            return false
+        }
+    }
+    return true
+}
+
+public extension Media {
+    func isLikelyToBeUpdated() -> Bool {
+        return false
+    }
+
+    func preventsAutomaticMessageSendingFailure() -> Bool {
+        return false
+    }
+
+    var indexableText: String? {
+        return nil
+    }
+}

submodules/Postbox/Sources/MediaBoxFile.swift

@@ -0,0 +1,760 @@
+import Foundation
+import SwiftSignalKit
+import ManagedFile
+import RangeSet
+
+private class MediaBoxPartialFileDataRequest {
+    let range: Range<Int64>
+    var waitingUntilAfterInitialFetch: Bool
+    let completion: (MediaResourceData) -> Void
+    
+    init(range: Range<Int64>, waitingUntilAfterInitialFetch: Bool, completion: @escaping (MediaResourceData) -> Void) {
+        self.range = range
+        self.waitingUntilAfterInitialFetch = waitingUntilAfterInitialFetch
+        self.completion = completion
+    }
+}
+
+final class MediaBoxPartialFile {
+    private let queue: Queue
+    private let manager: MediaBoxFileManager
+    private let storageBox: StorageBox
+    private let resourceId: Data
+    private let path: String
+    private let metaPath: String
+    private let completePath: String
+    private let completed: (Int64) -> Void
+    private let fd: MediaBoxFileManager.Item
+    fileprivate let fileMap: MediaBoxFileMap
+    private var dataRequests = Bag<MediaBoxPartialFileDataRequest>()
+    private let missingRanges: MediaBoxFileMissingRanges
+    private let rangeStatusRequests = Bag<((RangeSet<Int64>) -> Void, () -> Void)>()
+    private let statusRequests = Bag<((MediaResourceStatus) -> Void, Int64?)>()
+    
+    private let fullRangeRequests = Bag<Disposable>()
+    
+    private var currentFetch: (Promise<[(Range<Int64>, MediaBoxFetchPriority)]>, Disposable)?
+    private var processedAtLeastOneFetch: Bool = false
+    
+    init?(queue: Queue, manager: MediaBoxFileManager, storageBox: StorageBox, resourceId: Data, path: String, metaPath: String, completePath: String, completed: @escaping (Int64) -> Void) {
+        assert(queue.isCurrent())
+        self.manager = manager
+        self.storageBox = storageBox
+        self.resourceId = resourceId
+        
+        if let fd = manager.open(path: path, mode: .readwrite) {
+            self.queue = queue
+            self.path = path
+            self.metaPath = metaPath
+            self.completePath = completePath
+            self.completed = completed
+            self.fd = fd
+            if let fileMap = try? MediaBoxFileMap.read(manager: manager, path: self.metaPath) {
+                if !fileMap.ranges.isEmpty {
+                    let upperBound = fileMap.ranges.ranges.last!.upperBound
+                    if let actualSize = fileSize(path, useTotalFileAllocatedSize: false) {
+                        if upperBound > actualSize {
+                            self.fileMap = MediaBoxFileMap()
+                        } else {
+                            self.fileMap = fileMap
+                        }
+                    } else {
+                        self.fileMap = MediaBoxFileMap()
+                    }
+                } else {
+                    self.fileMap = fileMap
+                }
+            } else {
+                self.fileMap = MediaBoxFileMap()
+            }
+            self.storageBox.update(id: self.resourceId, size: self.fileMap.sum)
+            self.missingRanges = MediaBoxFileMissingRanges()
+        } else {
+            return nil
+        }
+    }
+    
+    deinit {
+        self.currentFetch?.1.dispose()
+    }
+    
+    static func extractPartialData(manager: MediaBoxFileManager, path: String, metaPath: String, range: Range<Int64>) -> Data? {
+        guard let fd = ManagedFile(queue: nil, path: path, mode: .read) else {
+            return nil
+        }
+        guard let fileMap = try? MediaBoxFileMap.read(manager: manager, path: metaPath) else {
+            return nil
+        }
+        guard let clippedRange = fileMap.contains(range) else {
+            return nil
+        }
+        let _ = fd.seek(position: Int64(clippedRange.lowerBound))
+        return fd.readData(count: Int(clippedRange.upperBound - clippedRange.lowerBound))
+    }
+    
+    static func internal_extractPartialData(manager: MediaBoxFileManager, path: String, metaPath: String, range: Range<Int64>) -> (file: ManagedFile, length: Int)? {
+        guard let fd = ManagedFile(queue: nil, path: path, mode: .read) else {
+            return nil
+        }
+        guard let fileMap = try? MediaBoxFileMap.read(manager: manager, path: metaPath) else {
+            return nil
+        }
+        guard let clippedRange = fileMap.contains(range) else {
+            return nil
+        }
+        let _ = fd.seek(position: Int64(clippedRange.lowerBound))
+        return (fd, Int(clippedRange.upperBound - clippedRange.lowerBound))
+    }
+    
+    static func internal_isPartialDataCached(manager: MediaBoxFileManager, path: String, metaPath: String, range: Range<Int64>) -> Bool {
+        guard let fileMap = try? MediaBoxFileMap.read(manager: manager, path: metaPath) else {
+            return false
+        }
+        guard let _ = fileMap.contains(range) else {
+            return false
+        }
+        return true
+    }
+    
+    var storedSize: Int64 {
+        assert(self.queue.isCurrent())
+        return self.fileMap.sum
+    }
+    
+    func reset() {
+        assert(self.queue.isCurrent())
+        
+        self.fileMap.reset()
+        self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+        
+        for request in self.dataRequests.copyItems() {
+            request.completion(MediaResourceData(path: self.path, offset: request.range.lowerBound, size: 0, complete: false))
+        }
+        
+        if let updatedRanges = self.missingRanges.reset(fileMap: self.fileMap) {
+            self.updateRequestRanges(updatedRanges, fetch: nil)
+        }
+        
+        if !self.rangeStatusRequests.isEmpty {
+            let ranges = self.fileMap.ranges
+            for (f, _) in self.rangeStatusRequests.copyItems() {
+                f(ranges)
+            }
+        }
+        
+        self.updateStatuses()
+    }
+    
+    func moveLocalFile(tempPath: String) {
+        assert(self.queue.isCurrent())
+        
+        do {
+            try FileManager.default.moveItem(atPath: tempPath, toPath: self.completePath)
+            
+            if let size = fileSize(self.completePath) {
+                unlink(self.path)
+                unlink(self.metaPath)
+                
+                for (_, completion) in self.missingRanges.clear() {
+                    completion()
+                }
+                
+                if let (_, disposable) = self.currentFetch {
+                    self.currentFetch = nil
+                    disposable.dispose()
+                }
+                
+                for request in self.dataRequests.copyItems() {
+                    request.completion(MediaResourceData(path: self.completePath, offset: request.range.lowerBound, size: max(0, size - request.range.lowerBound), complete: true))
+                }
+                self.dataRequests.removeAll()
+                
+                for statusRequest in self.statusRequests.copyItems() {
+                    statusRequest.0(.Local)
+                }
+                self.statusRequests.removeAll()
+                
+                self.storageBox.update(id: self.resourceId, size: self.fileMap.sum)
+                
+                self.completed(self.fileMap.sum)
+            } else {
+                assertionFailure()
+            }
+        } catch let e {
+            postboxLog("moveLocalFile error: \(e)")
+            assertionFailure()
+        }
+    }
+    
+    func copyLocalItem(_ item: MediaResourceDataFetchCopyLocalItem) {
+        assert(self.queue.isCurrent())
+        
+        do {
+            if item.copyTo(url: URL(fileURLWithPath: self.completePath)) {
+                
+            } else {
+                return
+            }
+            
+            if let size = fileSize(self.completePath) {
+                unlink(self.path)
+                unlink(self.metaPath)
+                
+                for (_, completion) in self.missingRanges.clear() {
+                    completion()
+                }
+                
+                if let (_, disposable) = self.currentFetch {
+                    self.currentFetch = nil
+                    disposable.dispose()
+                }
+                
+                for request in self.dataRequests.copyItems() {
+                    request.completion(MediaResourceData(path: self.completePath, offset: request.range.lowerBound, size: max(0, size - request.range.lowerBound), complete: true))
+                }
+                self.dataRequests.removeAll()
+                
+                for statusRequest in self.statusRequests.copyItems() {
+                    statusRequest.0(.Local)
+                }
+                self.statusRequests.removeAll()
+                
+                self.storageBox.update(id: self.resourceId, size: size)
+                
+                self.completed(size)
+            } else {
+                assertionFailure()
+            }
+        }
+    }
+    
+    func truncate(_ size: Int64) {
+        assert(self.queue.isCurrent())
+        
+        let range: Range<Int64> = size ..< Int64.max
+        
+        self.fileMap.truncate(size)
+        self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+        
+        self.checkDataRequestsAfterFill(range: range)
+    }
+    
+    func progressUpdated(_ progress: Float) {
+        assert(self.queue.isCurrent())
+        
+        self.fileMap.progressUpdated(progress)
+        self.updateStatuses()
+    }
+    
+    func write(offset: Int64, data: Data, dataRange: Range<Int64>) {
+        assert(self.queue.isCurrent())
+        
+        do {
+            try self.fd.access { fd in
+                let _ = fd.seek(position: offset)
+                let written = data.withUnsafeBytes { rawBytes -> Int in
+                    let bytes = rawBytes.baseAddress!.assumingMemoryBound(to: UInt8.self)
+
+                    return fd.write(bytes.advanced(by: Int(dataRange.lowerBound)), count: dataRange.count)
+                }
+                assert(written == dataRange.count)
+            }
+        } catch let e {
+            postboxLog("MediaBoxPartialFile.write error: \(e)")
+        }
+        
+        let range: Range<Int64> = offset ..< (offset + Int64(dataRange.count))
+        self.fileMap.fill(range)
+        self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+        
+        self.storageBox.update(id: self.resourceId, size: self.fileMap.sum)
+        
+        self.checkDataRequestsAfterFill(range: range)
+    }
+    
+    func checkDataRequestsAfterFill(range: Range<Int64>) {
+        var removeIndices: [(Int, MediaBoxPartialFileDataRequest)] = []
+        for (index, request) in self.dataRequests.copyItemsWithIndices() {
+            if request.range.overlaps(range) {
+                var maxValue = request.range.upperBound
+                if let truncationSize = self.fileMap.truncationSize {
+                    maxValue = truncationSize
+                }
+                if request.range.lowerBound > maxValue {
+                    assertionFailure()
+                    removeIndices.append((index, request))
+                } else {
+                    let intRange: Range<Int64> = request.range.lowerBound ..< min(maxValue, request.range.upperBound)
+                    if self.fileMap.ranges.isSuperset(of: RangeSet<Int64>(intRange)) {
+                        removeIndices.append((index, request))
+                    }
+                }
+            }
+        }
+        if !removeIndices.isEmpty {
+            for (index, request) in removeIndices {
+                self.dataRequests.remove(index)
+                var maxValue = request.range.upperBound
+                if let truncationSize = self.fileMap.truncationSize, truncationSize < maxValue {
+                    maxValue = truncationSize
+                }
+                request.completion(MediaResourceData(path: self.path, offset: request.range.lowerBound, size: maxValue - request.range.lowerBound, complete: true))
+            }
+        }
+        
+        var isCompleted = false
+        if let truncationSize = self.fileMap.truncationSize, let _ = self.fileMap.contains(0 ..< truncationSize) {
+            isCompleted = true
+        }
+        
+        if isCompleted {
+            for (_, completion) in self.missingRanges.clear() {
+                completion()
+            }
+        } else {
+            if let (updatedRanges, completions) = self.missingRanges.fill(range) {
+                self.updateRequestRanges(updatedRanges, fetch: nil)
+                completions.forEach({ $0() })
+            }
+        }
+        
+        if !self.rangeStatusRequests.isEmpty {
+            let ranges = self.fileMap.ranges
+            for (f, completed) in self.rangeStatusRequests.copyItems() {
+                f(ranges)
+                if isCompleted {
+                    completed()
+                }
+            }
+            if isCompleted {
+                self.rangeStatusRequests.removeAll()
+            }
+        }
+        
+        self.updateStatuses()
+        
+        if isCompleted {
+            for statusRequest in self.statusRequests.copyItems() {
+                statusRequest.0(.Local)
+            }
+            self.statusRequests.removeAll()
+            self.fd.sync()
+            let linkResult = link(self.path, self.completePath)
+            if linkResult != 0 {
+                //assert(linkResult == 0)
+            }
+            self.completed(self.fileMap.sum)
+        }
+    }
+    
+    func read(range: Range<Int64>) -> Data? {
+        assert(self.queue.isCurrent())
+        
+        if let actualRange = self.fileMap.contains(range) {
+            do {
+                var result: Data?
+                try self.fd.access { fd in
+                    let _ = fd.seek(position: Int64(actualRange.lowerBound))
+                    var data = Data(count: actualRange.count)
+                    let dataCount = data.count
+                    let readBytes = data.withUnsafeMutableBytes { rawBytes -> Int in
+                        let bytes = rawBytes.baseAddress!.assumingMemoryBound(to: Int8.self)
+                        return fd.read(bytes, dataCount)
+                    }
+                    if readBytes == data.count {
+                        result = data
+                    } else {
+                        result = nil
+                    }
+                }
+                return result
+            } catch let e {
+                postboxLog("MediaBoxPartialFile.read error: \(e)")
+                return nil
+            }
+        } else {
+            return nil
+        }
+    }
+    
+    func data(range: Range<Int64>, waitUntilAfterInitialFetch: Bool, next: @escaping (MediaResourceData) -> Void) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if let actualRange = self.fileMap.contains(range) {
+            next(MediaResourceData(path: self.path, offset: actualRange.lowerBound, size: Int64(actualRange.count), complete: true))
+            return EmptyDisposable
+        }
+        
+        var waitingUntilAfterInitialFetch = false
+        if waitUntilAfterInitialFetch && !self.processedAtLeastOneFetch {
+            waitingUntilAfterInitialFetch = true
+        } else {
+            next(MediaResourceData(path: self.path, offset: range.lowerBound, size: 0, complete: false))
+        }
+        
+        let index = self.dataRequests.add(MediaBoxPartialFileDataRequest(range: range, waitingUntilAfterInitialFetch: waitingUntilAfterInitialFetch, completion: { data in
+            next(data)
+        }))
+        
+        let queue = self.queue
+        return ActionDisposable { [weak self] in
+            queue.async {
+                if let strongSelf = self {
+                    strongSelf.dataRequests.remove(index)
+                }
+            }
+        }
+    }
+    
+    func fetched(range: Range<Int64>, priority: MediaBoxFetchPriority, fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>, error: @escaping (MediaResourceDataFetchError) -> Void, completed: @escaping () -> Void) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if let _ = self.fileMap.contains(range) {
+            completed()
+            return EmptyDisposable
+        }
+        
+        let (index, updatedRanges) = self.missingRanges.addRequest(fileMap: self.fileMap, range: range, priority: priority, error: error, completion: {
+            completed()
+        })
+        if let updatedRanges = updatedRanges {
+            self.updateRequestRanges(updatedRanges, fetch: fetch)
+        }
+        
+        let queue = self.queue
+        return ActionDisposable { [weak self] in
+            queue.async {
+                if let strongSelf = self {
+                    if let updatedRanges = strongSelf.missingRanges.removeRequest(fileMap: strongSelf.fileMap, index: index) {
+                        strongSelf.updateRequestRanges(updatedRanges, fetch: nil)
+                    }
+                }
+            }
+        }
+    }
+    
+    func fetchedFullRange(fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>, error: @escaping (MediaResourceDataFetchError) -> Void, completed: @escaping () -> Void) -> Disposable {
+        let queue = self.queue
+        let disposable = MetaDisposable()
+        
+        let index = self.fullRangeRequests.add(disposable)
+        self.updateStatuses()
+        
+        disposable.set(self.fetched(range: 0 ..< Int64.max, priority: .default, fetch: fetch, error: { e in
+            error(e)
+        }, completed: { [weak self] in
+            queue.async {
+                if let strongSelf = self {
+                    strongSelf.fullRangeRequests.remove(index)
+                    if strongSelf.fullRangeRequests.isEmpty {
+                        strongSelf.updateStatuses()
+                    }
+                }
+                completed()
+            }
+        }))
+        
+        return ActionDisposable { [weak self] in
+            queue.async {
+                if let strongSelf = self {
+                    strongSelf.fullRangeRequests.remove(index)
+                    disposable.dispose()
+                    if strongSelf.fullRangeRequests.isEmpty {
+                        strongSelf.updateStatuses()
+                    }
+                }
+            }
+        }
+    }
+    
+    func cancelFullRangeFetches() {
+        self.fullRangeRequests.copyItems().forEach({ $0.dispose() })
+        self.fullRangeRequests.removeAll()
+        
+        self.updateStatuses()
+    }
+    
+    private func updateStatuses() {
+        if !self.statusRequests.isEmpty {
+            for (f, size) in self.statusRequests.copyItems() {
+                let status = self.immediateStatus(size: size)
+                f(status)
+            }
+        }
+    }
+    
+    func rangeStatus(next: @escaping (RangeSet<Int64>) -> Void, completed: @escaping () -> Void) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        next(self.fileMap.ranges)
+        if let truncationSize = self.fileMap.truncationSize, let _ = self.fileMap.contains(0 ..< truncationSize) {
+            completed()
+            return EmptyDisposable
+        }
+        
+        let index = self.rangeStatusRequests.add((next, completed))
+        
+        let queue = self.queue
+        return ActionDisposable { [weak self] in
+            queue.async {
+                if let strongSelf = self {
+                    strongSelf.rangeStatusRequests.remove(index)
+                }
+            }
+        }
+    }
+    
+    private func immediateStatus(size: Int64?) -> MediaResourceStatus {
+        let status: MediaResourceStatus
+        if self.fullRangeRequests.isEmpty && self.currentFetch == nil {
+            if let truncationSize = self.fileMap.truncationSize, self.fileMap.sum == truncationSize {
+                status = .Local
+            } else {
+                let progress: Float
+                if let truncationSize = self.fileMap.truncationSize, truncationSize != 0 {
+                    progress = Float(self.fileMap.sum) / Float(truncationSize)
+                } else if let size = size {
+                    progress = Float(self.fileMap.sum) / Float(size)
+                } else {
+                    progress = self.fileMap.progress ?? 0.0
+                }
+                status = .Remote(progress: progress)
+            }
+        } else {
+            let progress: Float
+            if let truncationSize = self.fileMap.truncationSize, truncationSize != 0 {
+                progress = Float(self.fileMap.sum) / Float(truncationSize)
+            } else if let size = size {
+                progress = Float(self.fileMap.sum) / Float(size)
+            } else {
+                progress = self.fileMap.progress ?? 0.0
+            }
+            status = .Fetching(isActive: true, progress: progress)
+        }
+        return status
+    }
+    
+    func status(next: @escaping (MediaResourceStatus) -> Void, completed: @escaping () -> Void, size: Int64?) -> Disposable {
+        let index = self.statusRequests.add((next, size))
+        
+        let value = self.immediateStatus(size: size)
+        next(value)
+        if case .Local = value {
+            completed()
+            return EmptyDisposable
+        } else {
+            let queue = self.queue
+            return ActionDisposable { [weak self] in
+                queue.async {
+                    if let strongSelf = self {
+                        strongSelf.statusRequests.remove(index)
+                    }
+                }
+            }
+        }
+    }
+    
+    private func updateRequestRanges(_ intervals: [(Range<Int64>, MediaBoxFetchPriority)], fetch: ((Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>)?) {
+        assert(self.queue.isCurrent())
+        
+        #if DEBUG
+        for interval in intervals {
+            assert(!interval.0.isEmpty)
+        }
+        #endif
+        if intervals.isEmpty {
+            if let (_, disposable) = self.currentFetch {
+                self.currentFetch = nil
+                self.updateStatuses()
+                disposable.dispose()
+            }
+        } else {
+            if let (promise, _) = self.currentFetch {
+                promise.set(.single(intervals))
+            } else if let fetch = fetch {
+                let promise = Promise<[(Range<Int64>, MediaBoxFetchPriority)]>()
+                let disposable = MetaDisposable()
+                self.currentFetch = (promise, disposable)
+                self.updateStatuses()
+                disposable.set((fetch(promise.get())
+                |> deliverOn(self.queue)).start(next: { [weak self] data in
+                    if let strongSelf = self {
+                        switch data {
+                            case .reset:
+                                if !strongSelf.fileMap.ranges.isEmpty {
+                                    strongSelf.reset()
+                                }
+                            case let .resourceSizeUpdated(size):
+                                strongSelf.truncate(size)
+                            case let .dataPart(resourceOffset, data, range, complete):
+                                if !data.isEmpty {
+                                    strongSelf.write(offset: resourceOffset, data: data, dataRange: range)
+                                }
+                                if complete {
+                                    if let maxOffset = strongSelf.fileMap.ranges.ranges.reversed().first?.upperBound {
+                                        let maxValue = max(resourceOffset + Int64(range.count), Int64(maxOffset))
+                                        strongSelf.truncate(maxValue)
+                                    }
+                                }
+                            case let .replaceHeader(data, range):
+                                strongSelf.write(offset: 0, data: data, dataRange: range)
+                            case let .moveLocalFile(path):
+                                strongSelf.moveLocalFile(tempPath: path)
+                            case let .moveTempFile(file):
+                                strongSelf.moveLocalFile(tempPath: file.path)
+                                TempBox.shared.dispose(file)
+                            case let .copyLocalItem(item):
+                                strongSelf.copyLocalItem(item)
+                            case let .progressUpdated(progress):
+                                strongSelf.progressUpdated(progress)
+                        }
+                        if !strongSelf.processedAtLeastOneFetch {
+                            strongSelf.processedAtLeastOneFetch = true
+                            for request in strongSelf.dataRequests.copyItems() {
+                                if request.waitingUntilAfterInitialFetch {
+                                    request.waitingUntilAfterInitialFetch = false
+                                    
+                                    if let actualRange = strongSelf.fileMap.contains(request.range) {
+                                        request.completion(MediaResourceData(path: strongSelf.path, offset: actualRange.lowerBound, size: Int64(actualRange.count), complete: true))
+                                    } else {
+                                        request.completion(MediaResourceData(path: strongSelf.path, offset: request.range.lowerBound, size: 0, complete: false))
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }, error: { [weak self] e in
+                    guard let strongSelf = self else {
+                        return
+                    }
+                    for (error, _) in strongSelf.missingRanges.clear() {
+                        error(e)
+                    }
+                }))
+                promise.set(.single(intervals))
+            }
+        }
+    }
+}
+
+private final class MediaBoxFileMissingRange {
+    var range: Range<Int64>
+    let priority: MediaBoxFetchPriority
+    var remainingRanges: RangeSet<Int64>
+    let error: (MediaResourceDataFetchError) -> Void
+    let completion: () -> Void
+    
+    init(range: Range<Int64>, priority: MediaBoxFetchPriority, error: @escaping (MediaResourceDataFetchError) -> Void, completion: @escaping () -> Void) {
+        self.range = range
+        self.priority = priority
+        self.remainingRanges = RangeSet<Int64>(range)
+        self.error = error
+        self.completion = completion
+    }
+}
+
+private final class MediaBoxFileMissingRanges {
+    private var requestedRanges = Bag<MediaBoxFileMissingRange>()
+    
+    private var missingRangesFlattened = RangeSet<Int64>()
+    private var missingRangesByPriority: [MediaBoxFetchPriority: RangeSet<Int64>] = [:]
+    
+    func clear() -> [((MediaResourceDataFetchError) -> Void, () -> Void)] {
+        let errorsAndCompletions = self.requestedRanges.copyItems().map({ ($0.error, $0.completion) })
+        self.requestedRanges.removeAll()
+        return errorsAndCompletions
+    }
+    
+    func reset(fileMap: MediaBoxFileMap) -> [(Range<Int64>, MediaBoxFetchPriority)]? {
+        return self.update(fileMap: fileMap)
+    }
+    
+    private func missingRequestedIntervals() -> [(Range<Int64>, MediaBoxFetchPriority)] {
+        var intervalsByPriority: [MediaBoxFetchPriority: RangeSet<Int64>] = [:]
+        var remainingIntervals = RangeSet<Int64>()
+        for item in self.requestedRanges.copyItems() {
+            var requestedInterval = RangeSet<Int64>(item.range)
+            requestedInterval.formIntersection(self.missingRangesFlattened)
+            if !requestedInterval.isEmpty {
+                if intervalsByPriority[item.priority] == nil {
+                    intervalsByPriority[item.priority] = RangeSet<Int64>()
+                }
+                intervalsByPriority[item.priority]?.formUnion(requestedInterval)
+                remainingIntervals.formUnion(requestedInterval)
+            }
+        }
+        
+        var result: [(Range<Int64>, MediaBoxFetchPriority)] = []
+        
+        for priority in intervalsByPriority.keys.sorted(by: { $0.rawValue > $1.rawValue }) {
+            let currentIntervals = intervalsByPriority[priority]!.intersection(remainingIntervals)
+            remainingIntervals.subtract(currentIntervals)
+            for range in currentIntervals.ranges {
+                if !range.isEmpty {
+                    result.append((range, priority))
+                }
+            }
+        }
+        
+        return result
+    }
+    
+    func fill(_ range: Range<Int64>) -> ([(Range<Int64>, MediaBoxFetchPriority)], [() -> Void])? {
+        if self.missingRangesFlattened.intersects(range) {
+            self.missingRangesFlattened.remove(contentsOf: range)
+            for priority in self.missingRangesByPriority.keys {
+                self.missingRangesByPriority[priority]!.remove(contentsOf: range)
+            }
+            
+            var completions: [() -> Void] = []
+            for (index, item) in self.requestedRanges.copyItemsWithIndices() {
+                if item.range.overlaps(range) {
+                    item.remainingRanges.remove(contentsOf: range)
+                    if item.remainingRanges.isEmpty {
+                        self.requestedRanges.remove(index)
+                        completions.append(item.completion)
+                    }
+                }
+            }
+            
+            return (self.missingRequestedIntervals(), completions)
+        } else {
+            return nil
+        }
+    }
+    
+    func addRequest(fileMap: MediaBoxFileMap, range: Range<Int64>, priority: MediaBoxFetchPriority, error: @escaping (MediaResourceDataFetchError) -> Void, completion: @escaping () -> Void) -> (Int, [(Range<Int64>, MediaBoxFetchPriority)]?) {
+        let index = self.requestedRanges.add(MediaBoxFileMissingRange(range: range, priority: priority, error: error, completion: completion))
+        
+        return (index, self.update(fileMap: fileMap))
+    }
+    
+    func removeRequest(fileMap: MediaBoxFileMap, index: Int) -> [(Range<Int64>, MediaBoxFetchPriority)]? {
+        self.requestedRanges.remove(index)
+        return self.update(fileMap: fileMap)
+    }
+    
+    private func update(fileMap: MediaBoxFileMap) -> [(Range<Int64>, MediaBoxFetchPriority)]? {
+        var byPriority: [MediaBoxFetchPriority: RangeSet<Int64>] = [:]
+        var flattened = RangeSet<Int64>()
+        for item in self.requestedRanges.copyItems() {
+            let intRange: Range<Int64> = item.range
+            if byPriority[item.priority] == nil {
+                byPriority[item.priority] = RangeSet<Int64>()
+            }
+            byPriority[item.priority]!.insert(contentsOf: intRange)
+            flattened.insert(contentsOf: intRange)
+        }
+        for priority in byPriority.keys {
+            byPriority[priority]!.subtract(fileMap.ranges)
+        }
+        flattened.subtract(fileMap.ranges)
+        if byPriority != self.missingRangesByPriority {
+            self.missingRangesByPriority = byPriority
+            self.missingRangesFlattened = flattened
+            
+            return self.missingRequestedIntervals()
+        }
+        return nil
+    }
+}

submodules/Postbox/Sources/MediaBoxFileContext.swift

@@ -0,0 +1,20 @@
+import Foundation
+import SwiftSignalKit
+import RangeSet
+
+protocol MediaBoxFileContext: AnyObject {
+    var isEmpty: Bool { get }
+    
+    func addReference() -> Int
+    func removeReference(_ index: Int)
+    
+    func data(range: Range<Int64>, waitUntilAfterInitialFetch: Bool, next: @escaping (MediaResourceData) -> Void) -> Disposable
+    
+    func fetched(range: Range<Int64>, priority: MediaBoxFetchPriority, fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>, error: @escaping (MediaResourceDataFetchError) -> Void, completed: @escaping () -> Void) -> Disposable
+    func fetchedFullRange(fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>, error: @escaping (MediaResourceDataFetchError) -> Void, completed: @escaping () -> Void) -> Disposable
+    func cancelFullRangeFetches()
+    func rangeStatus(next: @escaping (RangeSet<Int64>) -> Void, completed: @escaping () -> Void) -> Disposable
+    func status(next: @escaping (MediaResourceStatus) -> Void, completed: @escaping () -> Void, size: Int64?) -> Disposable
+    
+    func internalStore(data: Data, range: Range<Int64>)
+}

submodules/Postbox/Sources/MediaBoxFileContextV2Impl.swift

@@ -0,0 +1,817 @@
+import Foundation
+import RangeSet
+import SwiftSignalKit
+
+public final class MediaBoxFileContextV2Impl: MediaBoxFileContext {
+    private final class RangeRequest {
+        let value: Range<Int64>
+        let priority: MediaBoxFetchPriority
+        let isFullRange: Bool
+        let error: (MediaResourceDataFetchError) -> Void
+        let completed: () -> Void
+        
+        init(
+            value: Range<Int64>,
+            priority: MediaBoxFetchPriority,
+            isFullRange: Bool,
+            error: @escaping (MediaResourceDataFetchError) -> Void,
+            completed: @escaping () -> Void
+        ) {
+            self.value = value
+            self.priority = priority
+            self.isFullRange = isFullRange
+            self.error = error
+            self.completed = completed
+        }
+    }
+    
+    private final class StatusRequest {
+        let size: Int64?
+        let next: (MediaResourceStatus) -> Void
+        let completed: () -> Void
+        
+        var reportedStatus: MediaResourceStatus?
+        
+        init(size: Int64?, next: @escaping (MediaResourceStatus) -> Void, completed: @escaping () -> Void) {
+            self.size = size
+            self.next = next
+            self.completed = completed
+        }
+    }
+    
+    private final class PartialDataRequest {
+        let range: Range<Int64>
+        let next: (MediaResourceData) -> Void
+        
+        var waitingUntilAfterInitialFetch: Bool
+        var reportedStatus: MediaResourceData?
+        
+        init(
+            range: Range<Int64>,
+            waitUntilAfterInitialFetch: Bool,
+            next: @escaping (MediaResourceData) -> Void
+        ) {
+            self.range = range
+            self.waitingUntilAfterInitialFetch = waitUntilAfterInitialFetch
+            self.next = next
+        }
+    }
+    
+    private final class RangeStatusRequest {
+        let next: (RangeSet<Int64>) -> Void
+        let completed: () -> Void
+        
+        var reportedStatus: RangeSet<Int64>?
+        
+        init(
+            next: @escaping (RangeSet<Int64>) -> Void,
+            completed: @escaping () -> Void
+        ) {
+            self.next = next
+            self.completed = completed
+        }
+    }
+    
+    private struct MaterializedRangeRequest: Equatable {
+        let range: Range<Int64>
+        let priority: MediaBoxFetchPriority
+        
+        init(
+            range: Range<Int64>,
+            priority: MediaBoxFetchPriority
+        ) {
+            self.range = range
+            self.priority = priority
+        }
+    }
+    
+    private final class PendingFetch {
+        let initialFilterRanges: RangeSet<Int64>
+        let ranges = Promise<[(Range<Int64>, MediaBoxFetchPriority)]>()
+        let disposable: Disposable
+        
+        init(initialFilterRanges: RangeSet<Int64>, disposable: Disposable) {
+            self.initialFilterRanges = initialFilterRanges
+            self.disposable = disposable
+        }
+    }
+    
+    private final class PartialState {
+        private let queue: Queue
+        private let manager: MediaBoxFileManager
+        private let storageBox: StorageBox?
+        private let resourceId: Data
+        private let partialPath: String
+        private let fullPath: String
+        private let metaPath: String
+        
+        private let destinationFile: MediaBoxFileManager.Item?
+        
+        private let fileMap: MediaBoxFileMap
+        private var rangeRequests = Bag<RangeRequest>()
+        private var statusRequests = Bag<StatusRequest>()
+        private var rangeStatusRequests = Bag<RangeStatusRequest>()
+        private var partialDataRequests = Bag<PartialDataRequest>()
+        
+        private var fetchImpl: ((Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>)?
+        
+        private var materializedRangeRequests: [MaterializedRangeRequest] = []
+        private var pendingFetch: PendingFetch?
+        private var hasPerformedAnyFetch: Bool = false
+        
+        private var isComplete: Bool = false
+        
+        init(
+            queue: Queue,
+            manager: MediaBoxFileManager,
+            storageBox: StorageBox?,
+            resourceId: Data,
+            partialPath: String,
+            fullPath: String,
+            metaPath: String
+        ) {
+            self.queue = queue
+            self.manager = manager
+            self.storageBox = storageBox
+            self.resourceId = resourceId
+            self.partialPath = partialPath
+            self.fullPath = fullPath
+            self.metaPath = metaPath
+            
+            if !FileManager.default.fileExists(atPath: self.partialPath) {
+                let _ = try? FileManager.default.removeItem(atPath: self.metaPath)
+                self.fileMap = MediaBoxFileMap()
+                self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+            } else {
+                do {
+                    self.fileMap = try MediaBoxFileMap.read(manager: self.manager, path: self.metaPath)
+                } catch {
+                    let _ = try? FileManager.default.removeItem(atPath: self.metaPath)
+                    self.fileMap = MediaBoxFileMap()
+                }
+            }
+            
+            self.destinationFile = self.manager.open(path: self.partialPath, mode: .readwrite)
+            
+            if FileManager.default.fileExists(atPath: self.fullPath) {
+                self.isComplete = true
+            }
+        }
+        
+        deinit {
+            self.pendingFetch?.disposable.dispose()
+        }
+        
+        func request(
+            range: Range<Int64>,
+            isFullRange: Bool,
+            priority: MediaBoxFetchPriority,
+            fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>,
+            error: @escaping (MediaResourceDataFetchError) -> Void,
+            completed: @escaping () -> Void
+        ) -> Disposable {
+            assert(self.queue.isCurrent())
+            
+            self.fetchImpl = fetch
+            
+            let request = RangeRequest(
+                value: range,
+                priority: priority,
+                isFullRange: isFullRange,
+                error: error,
+                completed: completed
+            )
+            if self.updateRangeRequest(request: request) {
+                if !self.isComplete, let truncationSize = self.fileMap.truncationSize, truncationSize == self.fileMap.sum {
+                    self.isComplete = true
+                    
+                    let linkResult = link(self.partialPath, self.fullPath)
+                    if linkResult != 0 {
+                        postboxLog("MediaBoxFileContextV2Impl: error while linking \(self.partialPath): \(linkResult)")
+                    }
+                }
+                
+                self.updateRequests()
+                
+                return EmptyDisposable
+            } else {
+                let index = self.rangeRequests.add(request)
+                
+                self.updateRequests()
+                
+                let queue = self.queue
+                return ActionDisposable { [weak self] in
+                    queue.async {
+                        guard let `self` = self else {
+                            return
+                        }
+                        self.rangeRequests.remove(index)
+                        self.updateRequests()
+                    }
+                }
+            }
+        }
+        
+        func cancelFullRangeFetches() {
+            for (index, rangeRequest) in self.rangeRequests.copyItemsWithIndices() {
+                if rangeRequest.isFullRange {
+                    self.rangeRequests.remove(index)
+                }
+            }
+            self.updateRequests()
+        }
+        
+        func status(next: @escaping (MediaResourceStatus) -> Void, completed: @escaping () -> Void, size: Int64?) -> Disposable {
+            assert(self.queue.isCurrent())
+            
+            let request = StatusRequest(
+                size: size,
+                next: next,
+                completed: completed
+            )
+            if self.updateStatusRequest(request: request) {
+                return EmptyDisposable
+            } else {
+                let index = self.statusRequests.add(request)
+                
+                let queue = self.queue
+                return ActionDisposable { [weak self] in
+                    queue.async {
+                        guard let `self` = self else {
+                            return
+                        }
+                        self.statusRequests.remove(index)
+                    }
+                }
+            }
+        }
+        
+        func partialData(range: Range<Int64>, waitUntilAfterInitialFetch: Bool, next: @escaping (MediaResourceData) -> Void) -> Disposable {
+            let request = PartialDataRequest(
+                range: range,
+                waitUntilAfterInitialFetch: waitUntilAfterInitialFetch && !self.hasPerformedAnyFetch,
+                next: next
+            )
+            if self.updatePartialDataRequest(request: request) {
+                return EmptyDisposable
+            } else {
+                let index = self.partialDataRequests.add(request)
+                
+                let queue = self.queue
+                return ActionDisposable { [weak self] in
+                    queue.async {
+                        guard let `self` = self else {
+                            return
+                        }
+                        self.partialDataRequests.remove(index)
+                    }
+                }
+            }
+        }
+        
+        func rangeStatus(
+            next: @escaping (RangeSet<Int64>) -> Void,
+            completed: @escaping () -> Void
+        ) -> Disposable {
+            assert(self.queue.isCurrent())
+            
+            let request = RangeStatusRequest(
+                next: next,
+                completed: completed
+            )
+            if self.updateRangeStatusRequest(request: request) {
+                return EmptyDisposable
+            } else {
+                let index = self.rangeStatusRequests.add(request)
+                
+                let queue = self.queue
+                return ActionDisposable { [weak self] in
+                    queue.async {
+                        guard let `self` = self else {
+                            return
+                        }
+                        self.rangeStatusRequests.remove(index)
+                    }
+                }
+            }
+        }
+        
+        func internalStore(data: Data, range: Range<Int64>) {
+            assert(self.queue.isCurrent())
+            
+            if data.count == Int(range.upperBound - range.lowerBound) {
+                self.processFetchResult(result: .dataPart(resourceOffset: range.lowerBound, data: data, range: 0 ..< Int64(data.count), complete: false))
+            } else {
+                assertionFailure()
+            }
+        }
+        
+        private func updateRequests() {
+            var rangesByPriority: [MediaBoxFetchPriority: RangeSet<Int64>] = [:]
+            for (index, rangeRequest) in self.rangeRequests.copyItemsWithIndices() {
+                if self.updateRangeRequest(request: rangeRequest) {
+                    self.rangeRequests.remove(index)
+                    continue
+                }
+                
+                if rangesByPriority[rangeRequest.priority] == nil {
+                    rangesByPriority[rangeRequest.priority] = RangeSet()
+                }
+                rangesByPriority[rangeRequest.priority]?.formUnion(RangeSet<Int64>(rangeRequest.value))
+            }
+            
+            let initialFilterRanges: RangeSet<Int64>
+            if let current = self.pendingFetch {
+                initialFilterRanges = current.initialFilterRanges
+            } else {
+                initialFilterRanges = self.fileMap.ranges
+            }
+            
+            var materializedRangeRequests: [MaterializedRangeRequest] = []
+            for (priority, ranges) in rangesByPriority.sorted(by: { $0.key.rawValue < $1.key.rawValue }) {
+                let filteredRanges = ranges.subtracting(initialFilterRanges)
+                for range in filteredRanges.ranges {
+                    materializedRangeRequests.append(MaterializedRangeRequest(range: range, priority: priority))
+                }
+            }
+            
+            if self.materializedRangeRequests != materializedRangeRequests {
+                self.materializedRangeRequests = materializedRangeRequests
+                
+                if !materializedRangeRequests.isEmpty {
+                    if let fetchImpl = self.fetchImpl {
+                        let pendingFetch: PendingFetch
+                        if let current = self.pendingFetch {
+                            pendingFetch = current
+                        } else {
+                            let disposable = MetaDisposable()
+                            pendingFetch = PendingFetch(initialFilterRanges: initialFilterRanges, disposable: disposable)
+                            self.pendingFetch = pendingFetch
+                            self.hasPerformedAnyFetch = true
+                            
+                            let queue = self.queue
+                            disposable.set(fetchImpl(pendingFetch.ranges.get()).startStrict(next: { [weak self] result in
+                                queue.async {
+                                    guard let `self` = self else {
+                                        return
+                                    }
+                                    self.processFetchResult(result: result)
+                                }
+                            }, error: { [weak self] error in
+                                queue.async {
+                                    guard let `self` = self else {
+                                        return
+                                    }
+                                    self.processFetchError(error: error)
+                                }
+                            }))
+                        }
+                        pendingFetch.ranges.set(.single(materializedRangeRequests.map { request -> (Range<Int64>, MediaBoxFetchPriority) in
+                            return (request.range, request.priority)
+                        }))
+                    }
+                } else {
+                    if let pendingFetch = self.pendingFetch {
+                        self.pendingFetch = nil
+                        pendingFetch.disposable.dispose()
+                    }
+                }
+            }
+            
+            self.updateStatusRequests()
+        }
+        
+        private func processFetchResult(result: MediaResourceDataFetchResult) {
+            assert(self.queue.isCurrent())
+            
+            switch result {
+            case let .dataPart(resourceOffset, data, dataRange, complete):
+                self.processWrite(resourceOffset: resourceOffset, data: data, dataRange: dataRange)
+                
+                if complete {
+                    if let maxOffset = self.fileMap.ranges.ranges.reversed().first?.upperBound {
+                        let maxValue = max(resourceOffset + Int64(dataRange.count), Int64(maxOffset))
+                        if self.fileMap.truncationSize != maxValue {
+                            self.fileMap.truncate(maxValue)
+                            self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+                        }
+                    }
+                }
+            case let .resourceSizeUpdated(size):
+                if self.fileMap.truncationSize != size {
+                    self.fileMap.truncate(size)
+                    self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+                }
+            case let .progressUpdated(progress):
+                self.fileMap.progressUpdated(progress)
+                self.updateStatusRequests()
+            case let .replaceHeader(data, range):
+                self.processWrite(resourceOffset: 0, data: data, dataRange: range)
+            case let .moveLocalFile(path):
+                do {
+                    try FileManager.default.moveItem(atPath: path, toPath: self.fullPath)
+                    self.processMovedFile()
+                } catch let e {
+                    postboxLog("MediaBoxFileContextV2Impl: error moving temp file at \(self.fullPath): \(e)")
+                }
+            case let .moveTempFile(file):
+                do {
+                    try FileManager.default.moveItem(atPath: file.path, toPath: self.fullPath)
+                    self.processMovedFile()
+                } catch let e {
+                    postboxLog("MediaBoxFileContextV2Impl: error moving temp file at \(self.fullPath): \(e)")
+                }
+                TempBox.shared.dispose(file)
+            case let .copyLocalItem(localItem):
+                do {
+                    if localItem.copyTo(url: URL(fileURLWithPath: self.fullPath)) {
+                        unlink(self.partialPath)
+                        unlink(self.metaPath)
+                    }
+                    self.processMovedFile()
+                }
+            case .reset:
+                if !self.fileMap.ranges.isEmpty {
+                    self.fileMap.reset()
+                    self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+                }
+                
+            }
+            
+            if !self.isComplete, let truncationSize = self.fileMap.truncationSize, truncationSize == self.fileMap.sum {
+                self.isComplete = true
+                
+                let linkResult = link(self.partialPath, self.fullPath)
+                if linkResult != 0 {
+                    postboxLog("MediaBoxFileContextV2Impl: error while linking \(self.partialPath): \(linkResult)")
+                }
+            }
+            
+            self.updateRequests()
+        }
+        
+        private func processWrite(resourceOffset: Int64, data: Data, dataRange: Range<Int64>) {
+            if let destinationFile = self.destinationFile {
+                do {
+                    var success = true
+                    try destinationFile.access { fd in
+                        if fd.seek(position: resourceOffset) {
+                            let written = data.withUnsafeBytes { rawBytes -> Int in
+                                let bytes = rawBytes.baseAddress!.assumingMemoryBound(to: UInt8.self)
+                                
+                                return fd.write(bytes.advanced(by: Int(dataRange.lowerBound)), count: dataRange.count)
+                            }
+                            assert(written == dataRange.count)
+                        } else {
+                            success = false
+                        }
+                    }
+                    if success {
+                        let range: Range<Int64> = resourceOffset ..< (resourceOffset + Int64(dataRange.count))
+                        self.fileMap.fill(range)
+                        self.fileMap.serialize(manager: self.manager, to: self.metaPath)
+                        
+                        self.storageBox?.update(id: self.resourceId, size: self.fileMap.sum)
+                    } else {
+                        postboxLog("MediaBoxFileContextV2Impl: error seeking file to \(resourceOffset) at \(self.partialPath)")
+                    }
+                } catch let e {
+                    postboxLog("MediaBoxFileContextV2Impl: error writing file at \(self.partialPath): \(e)")
+                }
+            }
+        }
+        
+        private func processMovedFile() {
+            if let size = fileSize(self.fullPath) {
+                self.isComplete = true
+                self.storageBox?.update(id: self.resourceId, size: size)
+            }
+        }
+        
+        private func processFetchError(error: MediaResourceDataFetchError) {
+            assert(self.queue.isCurrent())
+            
+            let rangeRequests = self.rangeRequests.copyItems()
+            self.rangeRequests.removeAll()
+            
+            self.statusRequests.removeAll()
+            self.rangeStatusRequests.removeAll()
+            
+            //TODO:set status to .remote?
+            
+            for rangeRequest in rangeRequests {
+                rangeRequest.error(error)
+            }
+        }
+        
+        private func updateRangeRequest(request: RangeRequest) -> Bool {
+            assert(self.queue.isCurrent())
+            
+            if self.fileMap.contains(request.value) != nil {
+                request.completed()
+                return true
+            } else {
+                return false
+            }
+        }
+        
+        private func updateStatusRequests() {
+            for (index, partialDataRequest) in self.partialDataRequests.copyItemsWithIndices() {
+                if self.updatePartialDataRequest(request: partialDataRequest) {
+                    self.partialDataRequests.remove(index)
+                }
+            }
+            for (index, statusRequest) in self.statusRequests.copyItemsWithIndices() {
+                if self.updateStatusRequest(request: statusRequest) {
+                    self.statusRequests.remove(index)
+                }
+            }
+            for (index, rangeStatusRequest) in self.rangeStatusRequests.copyItemsWithIndices() {
+                if self.updateRangeStatusRequest(request: rangeStatusRequest) {
+                    self.rangeStatusRequests.remove(index)
+                }
+            }
+        }
+        
+        private func updatePartialDataRequest(request: PartialDataRequest) -> Bool {
+            assert(self.queue.isCurrent())
+            
+            if self.isComplete, let size = fileSize(self.fullPath) {
+                var clampedLowerBound = request.range.lowerBound
+                var clampedUpperBound = request.range.upperBound
+                if clampedUpperBound > size {
+                    clampedUpperBound = size
+                }
+                if clampedLowerBound > clampedUpperBound {
+                    clampedLowerBound = clampedUpperBound
+                }
+                
+                let updatedStatus = MediaResourceData(path: self.fullPath, offset: clampedLowerBound, size: clampedUpperBound - clampedLowerBound, complete: true)
+                if request.reportedStatus != updatedStatus {
+                    request.reportedStatus = updatedStatus
+                    request.next(updatedStatus)
+                }
+                return true
+            } else if self.fileMap.contains(request.range) != nil {
+                let updatedStatus = MediaResourceData(path: self.partialPath, offset: request.range.lowerBound, size: request.range.upperBound - request.range.lowerBound, complete: true)
+                if request.reportedStatus != updatedStatus {
+                    request.reportedStatus = updatedStatus
+                    request.next(updatedStatus)
+                }
+                return true
+            } else {
+                let updatedStatus = MediaResourceData(path: self.partialPath, offset: request.range.lowerBound, size: 0, complete: false)
+                if request.reportedStatus != updatedStatus {
+                    if request.waitingUntilAfterInitialFetch {
+                        if self.hasPerformedAnyFetch {
+                            request.waitingUntilAfterInitialFetch = false
+                            request.reportedStatus = updatedStatus
+                            request.next(updatedStatus)
+                        }
+                    } else {
+                        request.reportedStatus = updatedStatus
+                        request.next(updatedStatus)
+                    }
+                }
+                return false
+            }
+        }
+        
+        private func updateStatusRequest(request: StatusRequest) -> Bool {
+            assert(self.queue.isCurrent())
+            
+            let updatedStatus: MediaResourceStatus
+            if self.isComplete {
+                updatedStatus = .Local
+            } else if let totalSize = self.fileMap.truncationSize ?? request.size {
+                let progress = Float(self.fileMap.sum) / Float(totalSize)
+                if self.pendingFetch != nil {
+                    updatedStatus = .Fetching(isActive: true, progress: progress)
+                } else {
+                    updatedStatus = .Remote(progress: progress)
+                }
+            } else if self.pendingFetch != nil {
+                if let progress = self.fileMap.progress {
+                    updatedStatus = .Fetching(isActive: true, progress: progress)
+                } else {
+                    updatedStatus = .Fetching(isActive: true, progress: 0.0)
+                }
+            } else {
+                updatedStatus = .Remote(progress: 0.0)
+            }
+            
+            if request.reportedStatus != updatedStatus {
+                request.reportedStatus = updatedStatus
+                request.next(updatedStatus)
+            }
+            
+            return false
+        }
+        
+        private func updateRangeStatusRequest(request: RangeStatusRequest) -> Bool {
+            assert(self.queue.isCurrent())
+            
+            let status: RangeSet<Int64>
+            if self.isComplete, let size = fileSize(self.fullPath) {
+                status = RangeSet(0 ..< size)
+            } else {
+                status = self.fileMap.ranges
+            }
+            if request.reportedStatus != status {
+                request.reportedStatus = status
+                request.next(status)
+                
+                if let truncationSize = self.fileMap.truncationSize, self.fileMap.sum == truncationSize {
+                    request.completed()
+                    return true
+                }
+            }
+            
+            return false
+        }
+    }
+    
+    private let queue: Queue
+    private let manager: MediaBoxFileManager
+    private let storageBox: StorageBox?
+    private let resourceId: Data
+    private let path: String
+    private let partialPath: String
+    private let metaPath: String
+    
+    private let references = Bag<Void>()
+    
+    private var partialState: PartialState?
+    
+    var isEmpty: Bool {
+        return self.references.isEmpty
+    }
+    
+    public init?(
+        queue: Queue,
+        manager: MediaBoxFileManager,
+        storageBox: StorageBox?,
+        resourceId: Data,
+        path: String,
+        partialPath: String,
+        metaPath: String
+    ) {
+        self.queue = queue
+        self.manager = manager
+        self.storageBox = storageBox
+        self.resourceId = resourceId
+        self.path = path
+        self.partialPath = partialPath
+        self.metaPath = metaPath
+    }
+    
+    func addReference() -> Int {
+        assert(self.queue.isCurrent())
+        return self.references.add(Void())
+    }
+    
+    func removeReference(_ index: Int) {
+        assert(self.queue.isCurrent())
+        return self.references.remove(index)
+    }
+    
+    private func withPartialState<T>(_ f: (PartialState) -> T) -> T {
+        if let partialState = self.partialState {
+            return f(partialState)
+        } else {
+            let partialState = PartialState(
+                queue: self.queue,
+                manager: self.manager,
+                storageBox: self.storageBox,
+                resourceId: self.resourceId,
+                partialPath: self.partialPath,
+                fullPath: self.path,
+                metaPath: self.metaPath
+            )
+            self.partialState = partialState
+            return f(partialState)
+        }
+    }
+    
+    public func data(range: Range<Int64>, waitUntilAfterInitialFetch: Bool, next: @escaping (MediaResourceData) -> Void) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if let size = fileSize(self.path) {
+            var clampedLowerBound = range.lowerBound
+            var clampedUpperBound = range.upperBound
+            if clampedUpperBound > size {
+                clampedUpperBound = size
+            }
+            if clampedLowerBound > clampedUpperBound {
+                clampedLowerBound = clampedUpperBound
+            }
+            next(MediaResourceData(path: self.path, offset: clampedLowerBound, size: clampedUpperBound - clampedLowerBound, complete: true))
+            return EmptyDisposable
+        } else {
+            return self.withPartialState { partialState in
+                return partialState.partialData(
+                    range: range,
+                    waitUntilAfterInitialFetch: waitUntilAfterInitialFetch,
+                    next: next
+                )
+            }
+        }
+    }
+    
+    public func fetched(
+        range: Range<Int64>,
+        priority: MediaBoxFetchPriority,
+        fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>,
+        error: @escaping (MediaResourceDataFetchError) -> Void,
+        completed: @escaping () -> Void
+    ) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if FileManager.default.fileExists(atPath: self.path) {
+            completed()
+            return EmptyDisposable
+        } else {
+            return self.withPartialState { partialState in
+                return partialState.request(
+                    range: range,
+                    isFullRange: false,
+                    priority: priority,
+                    fetch: fetch,
+                    error: error,
+                    completed: completed
+                )
+            }
+        }
+    }
+    
+    public func fetchedFullRange(
+        fetch: @escaping (Signal<[(Range<Int64>, MediaBoxFetchPriority)], NoError>) -> Signal<MediaResourceDataFetchResult, MediaResourceDataFetchError>,
+        error: @escaping (MediaResourceDataFetchError) -> Void,
+        completed: @escaping () -> Void
+    ) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if FileManager.default.fileExists(atPath: self.path) {
+            completed()
+            return EmptyDisposable
+        } else {
+            return self.withPartialState { partialState in
+                return partialState.request(
+                    range: 0 ..< Int64.max,
+                    isFullRange: true,
+                    priority: .default,
+                    fetch: fetch,
+                    error: error,
+                    completed: completed
+                )
+            }
+        }
+    }
+    
+    public func cancelFullRangeFetches() {
+        assert(self.queue.isCurrent())
+        
+        if let partialState = self.partialState {
+            partialState.cancelFullRangeFetches()
+        }
+    }
+    
+    public func rangeStatus(next: @escaping (RangeSet<Int64>) -> Void, completed: @escaping () -> Void) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if let size = fileSize(self.path) {
+            next(RangeSet<Int64>([0 ..< Int64(size) as Range<Int64>]))
+            completed()
+            
+            return EmptyDisposable
+        } else {
+            return self.withPartialState { partialState in
+                return partialState.rangeStatus(next: next, completed: completed)
+            }
+        }
+    }
+    
+    public func status(next: @escaping (MediaResourceStatus) -> Void, completed: @escaping () -> Void, size: Int64?) -> Disposable {
+        assert(self.queue.isCurrent())
+        
+        if let _ = fileSize(self.path) {
+            next(.Local)
+            completed()
+            
+            return EmptyDisposable
+        } else {
+            return self.withPartialState { partialState in
+                return partialState.status(next: next, completed: completed, size: size)
+            }
+        }
+    }
+    
+    public func internalStore(data: Data, range: Range<Int64>) {
+        if let _ = fileSize(self.path) {
+        } else {
+            self.withPartialState { partialState in
+                partialState.internalStore(data: data, range: range)
+            }
+        }
+    }
+}

submodules/Postbox/Sources/MediaBoxFileManager.swift

@@ -0,0 +1,173 @@
+import Foundation
+import SwiftSignalKit
+import ManagedFile
+
+public final class MediaBoxFileManager {
+    public enum Mode {
+        case read
+        case readwrite
+    }
+    
+    public enum AccessError: Error {
+        case generic
+    }
+    
+    final class Item {
+        final class Accessor {
+            private let file: ManagedFile
+            
+            init(file: ManagedFile) {
+                self.file = file
+            }
+            
+            func write(_ data: UnsafeRawPointer, count: Int) -> Int {
+                return self.file.write(data, count: count)
+            }
+            
+            func read(_ data: UnsafeMutableRawPointer, _ count: Int) -> Int {
+                return self.file.read(data, count)
+            }
+            
+            func readData(count: Int) -> Data {
+                return self.file.readData(count: count)
+            }
+            
+            func seek(position: Int64) -> Bool {
+                return self.file.seek(position: position)
+            }
+        }
+        
+        weak var manager: MediaBoxFileManager?
+        let path: String
+        let mode: Mode
+        
+        weak var context: ItemContext?
+        
+        init(manager: MediaBoxFileManager, path: String, mode: Mode) {
+            self.manager = manager
+            self.path = path
+            self.mode = mode
+        }
+        
+        deinit {
+            if let manager = self.manager, let context = self.context {
+                manager.discardItemContext(context: context)
+            }
+        }
+        
+        func access(_ f: (Accessor) throws -> Void) throws {
+            if let context = self.context {
+                try f(Accessor(file: context.file))
+            } else {
+                if let manager = self.manager {
+                    if let context = manager.takeContext(path: self.path, mode: self.mode) {
+                        self.context = context
+                        try f(Accessor(file: context.file))
+                    } else {
+                        throw AccessError.generic
+                    }
+                } else {
+                    throw AccessError.generic
+                }
+            }
+        }
+        
+        func sync() {
+            if let context = self.context {
+                context.sync()
+            }
+        }
+    }
+    
+    final class ItemContext {
+        let id: Int
+        let path: String
+        let mode: Mode
+        let file: ManagedFile
+        
+        private var isDisposed: Bool = false
+        
+        init?(id: Int, path: String, mode: Mode) {
+            let mappedMode: ManagedFile.Mode
+            switch mode {
+            case .read:
+                mappedMode = .read
+            case .readwrite:
+                mappedMode = .readwrite
+            }
+            
+            guard let file = ManagedFile(queue: nil, path: path, mode: mappedMode) else {
+                return nil
+            }
+            self.file = file
+            
+            self.id = id
+            self.path = path
+            self.mode = mode
+        }
+        
+        deinit {
+            assert(self.isDisposed)
+        }
+        
+        func dispose() {
+            if !self.isDisposed {
+                self.isDisposed = true
+                self.file._unsafeClose()
+            } else {
+                assertionFailure()
+            }
+        }
+        
+        func sync() {
+            self.file.sync()
+        }
+    }
+    
+    private let queue: Queue?
+    private var contexts: [Int: ItemContext] = [:]
+    private var nextItemId: Int = 0
+    private let maxOpenFiles: Int
+    
+    public init(queue: Queue?) {
+        self.queue = queue
+        self.maxOpenFiles = 16
+    }
+    
+    func open(path: String, mode: Mode) -> Item? {
+        if let queue = self.queue {
+            assert(queue.isCurrent())
+        }
+        
+        return Item(manager: self, path: path, mode: mode)
+    }
+    
+    private func takeContext(path: String, mode: Mode) -> ItemContext? {
+        if let queue = self.queue {
+            assert(queue.isCurrent())
+        }
+        
+        if self.contexts.count > self.maxOpenFiles {
+            if let minKey = self.contexts.keys.min(), let context = self.contexts[minKey] {
+                self.discardItemContext(context: context)
+            }
+        }
+        
+        let id = self.nextItemId
+        self.nextItemId += 1
+        let context = ItemContext(id: id, path: path, mode: mode)
+        self.contexts[id] = context
+        return context
+    }
+    
+    private func discardItemContext(context: ItemContext) {
+        if let queue = self.queue {
+            assert(queue.isCurrent())
+        }
+        
+        if let context = self.contexts.removeValue(forKey: context.id) {
+            context.dispose()
+        }
+    }
+}
+

submodules/Postbox/Sources/MediaBoxFileMap.swift

@@ -0,0 +1,275 @@
+import Foundation
+import RangeSet
+import Crc32
+
+final class MediaBoxFileMap {
+    enum FileMapError: Error {
+        case generic
+    }
+    
+    private(set) var sum: Int64
+    private(set) var ranges: RangeSet<Int64>
+    private(set) var truncationSize: Int64?
+    private(set) var progress: Float?
+
+    init() {
+        self.sum = 0
+        self.ranges = RangeSet<Int64>()
+        self.truncationSize = nil
+        self.progress = nil
+    }
+    
+    private init(
+        sum: Int64,
+        ranges: RangeSet<Int64>,
+        truncationSize: Int64?,
+        progress: Float?
+    ) {
+        self.sum = sum
+        self.ranges = ranges
+        self.truncationSize = truncationSize
+        self.progress = progress
+    }
+    
+    static func read(manager: MediaBoxFileManager, path: String) throws -> MediaBoxFileMap {
+        guard let length = fileSize(path) else {
+            throw FileMapError.generic
+        }
+        guard let fileItem = manager.open(path: path, mode: .readwrite) else {
+            throw FileMapError.generic
+        }
+        
+        var result: MediaBoxFileMap?
+        
+        try fileItem.access { fd in
+            var firstUInt32: UInt32 = 0
+            guard fd.read(&firstUInt32, 4) == 4 else {
+                throw FileMapError.generic
+            }
+            
+            if firstUInt32 == 0x7bac1487 {
+                var crc: UInt32 = 0
+                guard fd.read(&crc, 4) == 4 else {
+                    throw FileMapError.generic
+                }
+                
+                var count: Int32 = 0
+                var sum: Int64 = 0
+                var ranges = RangeSet<Int64>()
+                
+                guard fd.read(&count, 4) == 4 else {
+                    throw FileMapError.generic
+                }
+                
+                if count < 0 {
+                    throw FileMapError.generic
+                }
+                
+                if count < 0 || length < 4 + 4 + 4 + 8 + count * 2 * 8 {
+                    throw FileMapError.generic
+                }
+                
+                var truncationSizeValue: Int64 = 0
+                
+                var data = Data(count: Int(8 + count * 2 * 8))
+                let dataCount = data.count
+                if !(data.withUnsafeMutableBytes { rawBytes -> Bool in
+                    let bytes = rawBytes.baseAddress!.assumingMemoryBound(to: UInt8.self)
+
+                    guard fd.read(bytes, dataCount) == dataCount else {
+                        return false
+                    }
+                    
+                    memcpy(&truncationSizeValue, bytes, 8)
+                    
+                    let calculatedCrc = Crc32(bytes, Int32(dataCount))
+                    if calculatedCrc != crc {
+                        return false
+                    }
+                    
+                    var offset = 8
+                    for _ in 0 ..< count {
+                        var intervalOffset: Int64 = 0
+                        var intervalLength: Int64 = 0
+                        memcpy(&intervalOffset, bytes.advanced(by: offset), 8)
+                        memcpy(&intervalLength, bytes.advanced(by: offset + 8), 8)
+                        offset += 8 * 2
+                        
+                        ranges.insert(contentsOf: intervalOffset ..< (intervalOffset + intervalLength))
+                        
+                        sum += intervalLength
+                    }
+                    
+                    return true
+                }) {
+                    throw FileMapError.generic
+                }
+                
+                let mappedTruncationSize: Int64?
+                if truncationSizeValue == -1 {
+                    mappedTruncationSize = nil
+                } else if truncationSizeValue < 0 {
+                    mappedTruncationSize = nil
+                } else {
+                    mappedTruncationSize = truncationSizeValue
+                }
+
+                result = MediaBoxFileMap(
+                    sum: sum,
+                    ranges: ranges,
+                    truncationSize: mappedTruncationSize,
+                    progress: nil
+                )
+            } else {
+                let crc: UInt32 = firstUInt32
+                var count: Int32 = 0
+                var sum: Int32 = 0
+                var ranges = RangeSet<Int64>()
+                
+                guard fd.read(&count, 4) == 4 else {
+                    throw FileMapError.generic
+                }
+                
+                if count < 0 {
+                    throw FileMapError.generic
+                }
+                
+                if count < 0 || UInt64(length) < 4 + 4 + UInt64(count) * 2 * 4 {
+                    throw FileMapError.generic
+                }
+                
+                var truncationSizeValue: Int32 = 0
+                
+                var data = Data(count: Int(4 + count * 2 * 4))
+                let dataCount = data.count
+                if !(data.withUnsafeMutableBytes { rawBytes -> Bool in
+                    let bytes = rawBytes.baseAddress!.assumingMemoryBound(to: UInt8.self)
+
+                    guard fd.read(bytes, dataCount) == dataCount else {
+                        return false
+                    }
+                    
+                    memcpy(&truncationSizeValue, bytes, 4)
+                    
+                    let calculatedCrc = Crc32(bytes, Int32(dataCount))
+                    if calculatedCrc != crc {
+                        return false
+                    }
+                    
+                    var offset = 4
+                    for _ in 0 ..< count {
+                        var intervalOffset: Int32 = 0
+                        var intervalLength: Int32 = 0
+                        memcpy(&intervalOffset, bytes.advanced(by: offset), 4)
+                        memcpy(&intervalLength, bytes.advanced(by: offset + 4), 4)
+                        offset += 8
+                        
+                        ranges.insert(contentsOf: Int64(intervalOffset) ..< Int64(intervalOffset + intervalLength))
+                        
+                        sum += intervalLength
+                    }
+                    
+                    return true
+                }) {
+                    throw FileMapError.generic
+                }
+                
+                let mappedTruncationSize: Int64?
+                if truncationSizeValue == -1 {
+                    mappedTruncationSize = nil
+                } else {
+                    mappedTruncationSize = Int64(truncationSizeValue)
+                }
+                
+                result = MediaBoxFileMap(
+                    sum: Int64(sum),
+                    ranges: ranges,
+                    truncationSize: mappedTruncationSize,
+                    progress: nil
+                )
+            }
+        }
+        
+        guard let result = result else {
+            throw FileMapError.generic
+        }
+        return result
+    }
+    
+    func serialize(manager: MediaBoxFileManager, to path: String) {
+        guard let fileItem = manager.open(path: path, mode: .readwrite) else {
+            postboxLog("MediaBoxFile: serialize: cannot open file")
+            return
+        }
+        
+        let _ = try? fileItem.access { file in
+            let _ = file.seek(position: 0)
+            let buffer = WriteBuffer()
+            var magic: UInt32 = 0x7bac1487
+            buffer.write(&magic, offset: 0, length: 4)
+            
+            var zero: Int32 = 0
+            buffer.write(&zero, offset: 0, length: 4)
+            
+            let rangeView = self.ranges.ranges
+            var count: Int32 = Int32(rangeView.count)
+            buffer.write(&count, offset: 0, length: 4)
+            
+            var truncationSizeValue: Int64 = self.truncationSize ?? -1
+            buffer.write(&truncationSizeValue, offset: 0, length: 8)
+            
+            for range in rangeView {
+                var intervalOffset = range.lowerBound
+                var intervalLength = range.upperBound - range.lowerBound
+                buffer.write(&intervalOffset, offset: 0, length: 8)
+                buffer.write(&intervalLength, offset: 0, length: 8)
+            }
+            var crc: UInt32 = Crc32(buffer.memory.advanced(by: 4 + 4 + 4), Int32(buffer.length - (4 + 4 + 4)))
+            memcpy(buffer.memory.advanced(by: 4), &crc, 4)
+            let written = file.write(buffer.memory, count: buffer.length)
+            assert(written == buffer.length)
+        }
+    }
+    
+    func fill(_ range: Range<Int64>) {
+        var previousCount: Int64 = 0
+        for intersectionRange in self.ranges.intersection(RangeSet<Int64>(range)).ranges {
+            previousCount += intersectionRange.upperBound - intersectionRange.lowerBound
+        }
+        
+        self.ranges.insert(contentsOf: range)
+        self.sum += (range.upperBound - range.lowerBound) - previousCount
+    }
+    
+    func truncate(_ size: Int64) {
+        self.truncationSize = size
+    }
+    func progressUpdated(_ progress: Float) {
+        self.progress = progress
+    }
+    
+    func reset() {
+        self.truncationSize = nil
+        self.ranges = RangeSet<Int64>()
+        self.sum = 0
+        self.progress = nil
+    }
+    
+    func contains(_ range: Range<Int64>) -> Range<Int64>? {
+        let maxValue: Int64
+        if let truncationSize = self.truncationSize {
+            maxValue = truncationSize
+        } else {
+            maxValue = Int64.max
+        }
+        let clippedUpperBound = min(maxValue, range.upperBound)
+        let clippedRange: Range<Int64> = min(range.lowerBound, clippedUpperBound) ..< clippedUpperBound
+        let clippedRangeSet = RangeSet<Int64>(clippedRange)
+        
+        if self.ranges.isSuperset(of: clippedRangeSet) {
+            return clippedRange
+        } else {
+            return nil
+        }
+    }
+}

submodules/Postbox/Sources/MediaResource.swift

@@ -0,0 +1,66 @@
+import Foundation
+
+public struct MediaResourceId: Equatable, Hashable {
+    public var stringRepresentation: String
+
+    public init(_ stringRepresentation: String) {
+        self.stringRepresentation = stringRepresentation
+    }
+}
+
+public protocol MediaResource: AnyObject {
+    var id: MediaResourceId { get }
+    var size: Int64? { get }
+    var streamable: Bool { get }
+    var headerSize: Int32 { get }
+    
+    func isEqual(to: MediaResource) -> Bool
+}
+
+public extension MediaResource {
+    var streamable: Bool {
+        return false
+    }
+    
+    var headerSize: Int32 {
+        return 0
+    }
+}
+
+public protocol CachedMediaResourceRepresentation {
+    var uniqueId: String { get }
+    var keepDuration: CachedMediaRepresentationKeepDuration { get }
+    func isEqual(to: CachedMediaResourceRepresentation) -> Bool
+}
+
+public protocol MediaResourceFetchTag {
+}
+
+public protocol MediaResourceFetchInfo {
+}
+
+public final class MediaResourceStorageLocation {
+    public let peerId: PeerId
+    public let messageId: MessageId?
+    
+    public init(peerId: PeerId, messageId: MessageId?) {
+        self.peerId = peerId
+        self.messageId = messageId
+    }
+}
+
+public struct MediaResourceFetchParameters {
+    public let tag: MediaResourceFetchTag?
+    public let info: MediaResourceFetchInfo?
+    public let location: MediaResourceStorageLocation?
+    public let contentType: UInt8
+    public let isRandomAccessAllowed: Bool
+    
+    public init(tag: MediaResourceFetchTag?, info: MediaResourceFetchInfo?, location: MediaResourceStorageLocation?, contentType: UInt8, isRandomAccessAllowed: Bool) {
+        self.tag = tag
+        self.info = info
+        self.location = location
+        self.contentType = contentType
+        self.isRandomAccessAllowed = isRandomAccessAllowed
+    }
+}

submodules/Postbox/Sources/MediaResourceStatus.swift

@@ -0,0 +1,9 @@
+import Foundation
+import SwiftSignalKit
+
+public enum MediaResourceStatus: Equatable {
+    case Remote(progress: Float)
+    case Local
+    case Fetching(isActive: Bool, progress: Float)
+    case Paused(progress: Float)
+}

submodules/Postbox/Sources/MessageCustomTagHoleIndexTable.swift

@@ -0,0 +1,311 @@
+import Foundation
+
+private func decomposeKey(_ key: ValueBoxKey) -> (id: MessageId, threadId: Int64?, tag: Int32) {
+    let threadId = key.getInt64(8)
+    return (
+        MessageId(
+            peerId: PeerId(key.getInt64(0)),
+            namespace: key.getInt32(8 + 8 + 4),
+            id: key.getInt32(8 + 8 + 4 + 4)
+        ),
+        threadId == 0 ? nil : threadId,
+        key.getInt32(8 + 8)
+    )
+}
+
+private func decodeValue(value: ReadBuffer, peerId: PeerId, namespace: MessageId.Namespace) -> MessageId {
+    var id: Int32 = 0
+    value.read(&id, offset: 0, length: 4)
+    return MessageId(peerId: peerId, namespace: namespace, id: id)
+}
+
+final class MessageCustomTagHoleIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let seedConfiguration: SeedConfiguration
+    private let metadataTable: MessageHistoryMetadataTable
+    private let tagIdTable: MessageCustomTagIdTable
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, seedConfiguration: SeedConfiguration, metadataTable: MessageHistoryMetadataTable, tagIdTable: MessageCustomTagIdTable) {
+        self.seedConfiguration = seedConfiguration
+        self.metadataTable = metadataTable
+        self.tagIdTable = tagIdTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(id: MessageId, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4)
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setInt32(8 + 8 + 4, value: id.namespace)
+        key.setInt32(8 + 8 + 4 + 4, value: id.id)
+        return key
+    }
+    
+    private func lowerBound(peerId: PeerId, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId, threadId: threadId, tag: tag).successor
+    }
+    
+    private func lowerBound(peerId: PeerId, namespace: MessageId.Namespace, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setInt32(8 + 8 + 4, value: namespace)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, namespace: MessageId.Namespace, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: tag).successor
+    }
+    
+    private func ensureInitialized(peerId: PeerId, threadId: Int64?, tag: Int32, tagValue: MemoryBuffer) {
+        if !self.metadataTable.isPeerCustomTagInitialized(peerId: peerId, threadId: threadId, tag: tag) {
+            self.metadataTable.setPeerCustomTagInitialized(peerId: peerId, threadId: threadId, tag: tag)
+            
+            if let namespaces = self.seedConfiguration.messageThreadHoles(peerId.namespace, threadId) {
+                for namespace in namespaces {
+                    var operations: [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]] = [:]
+                    self.addInternal(peerId: peerId, threadId: threadId, tag: tag, tagValue: tagValue, namespace: namespace, range: 1 ... (Int32.max - 1), operations: &operations)
+                }
+            }
+        }
+    }
+    
+    func existingNamespaces(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer) -> Set<MessageId.Namespace> {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag)
+        
+        var result = Set<MessageId.Namespace>()
+        var currentLowerBound = self.lowerBound(peerId: peerId, threadId: threadId, tag: mappedTag)
+        let upperBound = self.upperBound(peerId: peerId, threadId: threadId, tag: mappedTag)
+        while true {
+            var decomposedKey: (id: MessageId, threadId: Int64?, tag: Int32)?
+            self.valueBox.range(self.table, start: currentLowerBound, end: upperBound, keys: { key in
+                decomposedKey = decomposeKey(key)
+                return false
+            }, limit: 1)
+            if let decomposedKey {
+                result.insert(decomposedKey.id.namespace)
+                currentLowerBound = self.upperBound(peerId: peerId, namespace: decomposedKey.id.namespace, threadId: threadId, tag: mappedTag)
+            } else {
+                break
+            }
+        }
+        return result
+    }
+    
+    func closest(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>) -> IndexSet {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag)
+        
+        var result = IndexSet()
+        
+        func processIntersectingRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == mappedTag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if holeRange.overlaps(range) {
+                result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+            }
+        }
+        
+        func processEdgeRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == mappedTag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+        }
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), threadId: threadId, tag: mappedTag).predecessor, end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: mappedTag).successor, values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: mappedTag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: mappedTag), values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 1)
+        
+        if !result.contains(Int(range.lowerBound)) {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), threadId: threadId, tag: mappedTag), end: self.lowerBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: mappedTag), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        if !result.contains(Int(range.upperBound)) {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: mappedTag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: mappedTag), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        
+        return result
+    }
+    
+    func add(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag)
+        
+        self.addInternal(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, namespace: namespace, range: range, operations: &operations)
+    }
+    
+    private func addInternal(peerId: PeerId, threadId: Int64?, tag: Int32, tagValue: MemoryBuffer, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let clippedLowerBound = max(1, range.lowerBound)
+        let clippedUpperBound = min(Int32.max - 1, range.upperBound)
+        if clippedLowerBound > clippedUpperBound {
+            return
+        }
+        let clippedRange = clippedLowerBound ... clippedUpperBound
+        
+        var insertedIndices = IndexSet()
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        var alreadyMapped = false
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == tag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<Int32> = lowerId.id ... upperId.id
+            if clippedRange.lowerBound >= holeRange.lowerBound && clippedRange.upperBound <= holeRange.upperBound {
+                alreadyMapped = true
+                return
+            } else if clippedRange.overlaps(holeRange) || (holeRange.upperBound != Int32.max && clippedRange.lowerBound == holeRange.upperBound + 1) || clippedRange.upperBound == holeRange.lowerBound - 1 {
+                removeKeys.append(upperId.id)
+                let unionRange: ClosedRange = min(clippedRange.lowerBound, holeRange.lowerBound) ... max(clippedRange.upperBound, holeRange.upperBound)
+                insertRanges.insert(integersIn: Int(unionRange.lowerBound) ... Int(unionRange.upperBound))
+            }
+        }
+        
+        let lowerScanBound = max(0, clippedRange.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), threadId: threadId, tag: tag), end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), threadId: threadId, tag: tag).successor, values: { key, value in
+            processRange(key, value)
+            if alreadyMapped {
+                return false
+            }
+            return true
+        }, limit: 0)
+        
+        if !alreadyMapped {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), threadId: threadId, tag: tag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: tag), values: { key, value in
+                processRange(key, value)
+                if alreadyMapped {
+                    return false
+                }
+                return true
+            }, limit: 1)
+        }
+        
+        if alreadyMapped {
+            return
+        }
+        
+        insertRanges.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        insertedIndices.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: id), threadId: threadId, tag: tag), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), threadId: threadId, tag: tag), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        addMessageHistoryHoleOperation(.insert(clippedRange), peerId: peerId, threadId: threadId, namespace: namespace, space: .customTag(tagValue, nil), to: &operations)
+    }
+    
+    func remove(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag)
+        
+        self.removeInternal(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, namespace: namespace, range: range, operations: &operations)
+    }
+    
+    private func removeInternal(peerId: PeerId, threadId: Int64?, tag: Int32, tagValue: MemoryBuffer, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == tag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if range.lowerBound <= holeRange.lowerBound && range.upperBound >= holeRange.upperBound {
+                removeKeys.append(upperId.id)
+            } else if range.overlaps(holeRange) {
+                removeKeys.append(upperId.id)
+                var holeIndices = IndexSet(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+                holeIndices.remove(integersIn: Int(range.lowerBound) ... Int(range.upperBound))
+                insertRanges.formUnion(holeIndices)
+            }
+        }
+        
+        let lowerScanBound = max(0, range.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), threadId: threadId, tag: tag), end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: tag).successor, values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: tag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: tag), values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 1)
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: id), threadId: threadId, tag: tag), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), threadId: threadId, tag: tag), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        if !removeKeys.isEmpty {
+            addMessageHistoryHoleOperation(.remove(range), peerId: peerId, threadId: threadId, namespace: namespace, space: .customTag(tagValue, nil), to: &operations)
+        }
+    }
+    
+    func resetAll() {
+        self.clearMemoryCache()
+        
+        self.valueBox.removeAllFromTable(self.table)
+        self.metadataTable.removePeerCustomTagInitializedList()
+    }
+}

submodules/Postbox/Sources/MessageCustomTagIdTable.swift

@@ -0,0 +1,46 @@
+import Foundation
+
+final class MessageCustomTagIdTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let metadataTable: MessageHistoryMetadataTable
+    
+    private var cachedIds: [MemoryBuffer: Int32] = [:]
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, metadataTable: MessageHistoryMetadataTable) {
+        self.metadataTable = metadataTable
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    func get(tag: MemoryBuffer) -> Int32 {
+        if let value = self.cachedIds[tag] {
+            return value
+        } else if let value = self.valueBox.get(self.table, key: ValueBoxKey(tag)) {
+            assert(value.length == 4)
+            var result: Int32 = 0
+            value.read(&result, offset: 0, length: 4)
+            self.cachedIds[tag] = result
+            return result
+        } else {
+            let id = self.metadataTable.getNextCustomTagIdAndIncrement()
+            
+            if self.useCaches {
+                self.cachedIds[tag] = id
+            }
+            
+            var storeId: Int32 = id
+            self.valueBox.set(self.table, key: ValueBoxKey(tag), value: MemoryBuffer(memory: &storeId, capacity: 4, length: 4, freeWhenDone: false))
+            
+            return id
+        }
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedIds.removeAll()
+    }
+    
+    override func beforeCommit() {
+    }
+}

submodules/Postbox/Sources/MessageCustomTagTable.swift

@@ -0,0 +1,128 @@
+import Foundation
+
+private func extractKey(_ key: ValueBoxKey) -> MessageIndex {
+    return MessageIndex(
+        id: MessageId(
+            peerId: PeerId(key.getInt64(0)),
+            namespace: key.getInt32(8 + 8 + 4),
+            id: key.getInt32(8 + 8 + 4 + 4 + 4)
+        ),
+        timestamp: key.getInt32(8 + 8 + 4 + 4)
+    )
+}
+
+class MessageCustomTagTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let messageCustomTagIdTable: MessageCustomTagIdTable
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4)
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, messageCustomTagIdTable: MessageCustomTagIdTable) {
+        self.messageCustomTagIdTable = messageCustomTagIdTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(threadId: Int64?, tag: Int32, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey {
+        key.setInt64(0, value: index.id.peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setInt32(8 + 8 + 4, value: index.id.namespace)
+        key.setInt32(8 + 8 + 4 + 4, value: index.timestamp)
+        key.setInt32(8 + 8 + 4 + 4 + 4, value: index.id.id)
+        return key
+    }
+    
+    private func lowerBound(threadId: Int64?, tag: Int32, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setInt32(8 + 8 + 4, value: namespace)
+        return key
+    }
+    
+    private func upperBound(threadId: Int64?, tag: Int32, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        return self.lowerBound(threadId: threadId, tag: tag, peerId: peerId, namespace: namespace).successor
+    }
+    
+    func add(threadId: Int64?, tag: MemoryBuffer, index: MessageIndex) {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        self.valueBox.set(self.table, key: self.key(threadId: threadId, tag: mappedTag, index: index, key: self.sharedKey), value: MemoryBuffer())
+    }
+    
+    func remove(threadId: Int64?, tag: MemoryBuffer, index: MessageIndex) {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        self.valueBox.remove(self.table, key: self.key(threadId: threadId, tag: mappedTag, index: index, key: self.sharedKey), secure: false)
+    }
+    
+    func entryExists(threadId: Int64?, tag: MemoryBuffer, index: MessageIndex) -> Bool {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        return self.valueBox.exists(self.table, key: self.key(threadId: threadId, tag: mappedTag, index: index, key: self.sharedKey))
+    }
+    
+    func earlierIndices(threadId: Int64?, tag: MemoryBuffer, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, minIndex: MessageIndex? = nil, count: Int) -> [MessageIndex] {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(threadId: threadId, tag: mappedTag, index: index).successor
+            } else {
+                key = self.key(threadId: threadId, tag: mappedTag, index: index)
+            }
+        } else {
+            key = self.upperBound(threadId: threadId, tag: mappedTag, peerId: peerId, namespace: namespace)
+        }
+        let endKey: ValueBoxKey
+        if let minIndex = minIndex {
+            endKey = self.key(threadId: threadId, tag: mappedTag, index: minIndex)
+        } else {
+            endKey = self.lowerBound(threadId: threadId, tag: mappedTag, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: endKey, keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func laterIndices(threadId: Int64?, tag: MemoryBuffer, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, count: Int) -> [MessageIndex] {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(threadId: threadId, tag: mappedTag, index: index).predecessor
+            } else {
+                key = self.key(threadId: threadId, tag: mappedTag, index: index)
+            }
+        } else {
+            key = self.lowerBound(threadId: threadId, tag: mappedTag, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: self.upperBound(threadId: threadId, tag: mappedTag, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func latestIndex(threadId: Int64?, tag: MemoryBuffer, peerId: PeerId, namespace: MessageId.Namespace) -> MessageIndex? {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        var result: MessageIndex?
+        self.valueBox.range(self.table, start: self.lowerBound(threadId: threadId, tag: mappedTag, peerId: peerId, namespace: namespace), end: self.upperBound(threadId: threadId, tag: mappedTag, peerId: peerId, namespace: namespace), keys: { key in
+            result = extractKey(key)
+            return true
+        }, limit: 1)
+        return result
+    }
+}

submodules/Postbox/Sources/MessageCustomTagWithTagHoleIndexTable.swift

@@ -0,0 +1,319 @@
+import Foundation
+
+private func decomposeKey(_ key: ValueBoxKey) -> (id: MessageId, threadId: Int64?, tag: Int32, regularTag: UInt32) {
+    let threadId = key.getInt64(8)
+    return (
+        MessageId(
+            peerId: PeerId(key.getInt64(0)),
+            namespace: key.getInt32(8 + 8 + 4 + 4),
+            id: key.getInt32(8 + 8 + 4 + 4 + 4)
+        ),
+        threadId == 0 ? nil : threadId,
+        key.getInt32(8 + 8),
+        key.getUInt32(8 + 8 + 4)
+    )
+}
+
+private func decodeValue(value: ReadBuffer, peerId: PeerId, namespace: MessageId.Namespace) -> MessageId {
+    var id: Int32 = 0
+    value.read(&id, offset: 0, length: 4)
+    return MessageId(peerId: peerId, namespace: namespace, id: id)
+}
+
+final class MessageCustomTagWithTagHoleIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let seedConfiguration: SeedConfiguration
+    private let metadataTable: MessageHistoryMetadataTable
+    private let tagIdTable: MessageCustomTagIdTable
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, seedConfiguration: SeedConfiguration, metadataTable: MessageHistoryMetadataTable, tagIdTable: MessageCustomTagIdTable) {
+        self.seedConfiguration = seedConfiguration
+        self.metadataTable = metadataTable
+        self.tagIdTable = tagIdTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(id: MessageId, threadId: Int64?, tag: Int32, regularTag: UInt32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4)
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setUInt32(8 + 8 + 4, value: regularTag)
+        key.setInt32(8 + 8 + 4 + 4, value: id.namespace)
+        key.setInt32(8 + 8 + 4 + 4 + 4, value: id.id)
+        return key
+    }
+    
+    private func lowerBound(peerId: PeerId, threadId: Int64?, tag: Int32, regularTag: UInt32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setUInt32(8 + 8 + 4, value: regularTag)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, threadId: Int64?, tag: Int32, regularTag: UInt32) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId, threadId: threadId, tag: tag, regularTag: regularTag).successor
+    }
+    
+    private func lowerBound(peerId: PeerId, namespace: MessageId.Namespace, threadId: Int64?, tag: Int32, regularTag: UInt32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setUInt32(8 + 8 + 4, value: regularTag)
+        key.setInt32(8 + 8 + 4 + 4, value: namespace)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, namespace: MessageId.Namespace, threadId: Int64?, tag: Int32, regularTag: UInt32) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: tag, regularTag: regularTag).successor
+    }
+    
+    private func ensureInitialized(peerId: PeerId, threadId: Int64?, tag: Int32, tagValue: MemoryBuffer, regularTag: UInt32) {
+        if !self.metadataTable.isPeerCustomTagInitialized(peerId: peerId, threadId: threadId, tag: tag, regularTag: regularTag) {
+            self.metadataTable.setPeerCustomTagInitialized(peerId: peerId, threadId: threadId, tag: tag, regularTag: regularTag)
+            
+            if let namespaces = self.seedConfiguration.messageThreadHoles(peerId.namespace, threadId) {
+                for namespace in namespaces {
+                    var operations: [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]] = [:]
+                    self.addInternal(peerId: peerId, threadId: threadId, tag: tag, tagValue: tagValue, regularTag: regularTag, namespace: namespace, range: 1 ... (Int32.max - 1), operations: &operations)
+                }
+            }
+        }
+    }
+    
+    func existingNamespaces(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32) -> Set<MessageId.Namespace> {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, regularTag: regularTag)
+        
+        var result = Set<MessageId.Namespace>()
+        var currentLowerBound = self.lowerBound(peerId: peerId, threadId: threadId, tag: mappedTag, regularTag: regularTag)
+        let upperBound = self.upperBound(peerId: peerId, threadId: threadId, tag: mappedTag, regularTag: regularTag)
+        while true {
+            var decomposedKey: (id: MessageId, threadId: Int64?, tag: Int32, regularTag: UInt32)?
+            self.valueBox.range(self.table, start: currentLowerBound, end: upperBound, keys: { key in
+                decomposedKey = decomposeKey(key)
+                return false
+            }, limit: 1)
+            if let decomposedKey {
+                result.insert(decomposedKey.id.namespace)
+                currentLowerBound = self.upperBound(peerId: peerId, namespace: decomposedKey.id.namespace, threadId: threadId, tag: mappedTag, regularTag: regularTag)
+            } else {
+                break
+            }
+        }
+        return result
+    }
+    
+    func closest(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>) -> IndexSet {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, regularTag: regularTag)
+        
+        var result = IndexSet()
+        
+        func processIntersectingRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag, keyRegularTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == mappedTag)
+            assert(keyRegularTag == regularTag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if holeRange.overlaps(range) {
+                result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+            }
+        }
+        
+        func processEdgeRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag, keyRegularTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == mappedTag)
+            assert(keyRegularTag == regularTag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+        }
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), threadId: threadId, tag: mappedTag, regularTag: regularTag).predecessor, end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: mappedTag, regularTag: regularTag).successor, values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: mappedTag, regularTag: regularTag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: mappedTag, regularTag: regularTag), values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 1)
+        
+        if !result.contains(Int(range.lowerBound)) {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), threadId: threadId, tag: mappedTag, regularTag: regularTag), end: self.lowerBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: mappedTag, regularTag: regularTag), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        if !result.contains(Int(range.upperBound)) {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: mappedTag, regularTag: regularTag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: mappedTag, regularTag: regularTag), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        
+        return result
+    }
+    
+    func add(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, regularTag: regularTag)
+        
+        self.addInternal(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, regularTag: regularTag, namespace: namespace, range: range, operations: &operations)
+    }
+    
+    private func addInternal(peerId: PeerId, threadId: Int64?, tag: Int32, tagValue: MemoryBuffer, regularTag: UInt32, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let clippedLowerBound = max(1, range.lowerBound)
+        let clippedUpperBound = min(Int32.max - 1, range.upperBound)
+        if clippedLowerBound > clippedUpperBound {
+            return
+        }
+        let clippedRange = clippedLowerBound ... clippedUpperBound
+        
+        var insertedIndices = IndexSet()
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        var alreadyMapped = false
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag, keyRegularTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == tag)
+            assert(keyRegularTag == regularTag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<Int32> = lowerId.id ... upperId.id
+            if clippedRange.lowerBound >= holeRange.lowerBound && clippedRange.upperBound <= holeRange.upperBound {
+                alreadyMapped = true
+                return
+            } else if clippedRange.overlaps(holeRange) || (holeRange.upperBound != Int32.max && clippedRange.lowerBound == holeRange.upperBound + 1) || clippedRange.upperBound == holeRange.lowerBound - 1 {
+                removeKeys.append(upperId.id)
+                let unionRange: ClosedRange = min(clippedRange.lowerBound, holeRange.lowerBound) ... max(clippedRange.upperBound, holeRange.upperBound)
+                insertRanges.insert(integersIn: Int(unionRange.lowerBound) ... Int(unionRange.upperBound))
+            }
+        }
+        
+        let lowerScanBound = max(0, clippedRange.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), threadId: threadId, tag: tag, regularTag: regularTag), end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), threadId: threadId, tag: tag, regularTag: regularTag).successor, values: { key, value in
+            processRange(key, value)
+            if alreadyMapped {
+                return false
+            }
+            return true
+        }, limit: 0)
+        
+        if !alreadyMapped {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), threadId: threadId, tag: tag, regularTag: regularTag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: tag, regularTag: regularTag), values: { key, value in
+                processRange(key, value)
+                if alreadyMapped {
+                    return false
+                }
+                return true
+            }, limit: 1)
+        }
+        
+        if alreadyMapped {
+            return
+        }
+        
+        insertRanges.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        insertedIndices.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: id), threadId: threadId, tag: tag, regularTag: regularTag), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), threadId: threadId, tag: tag, regularTag: regularTag), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        addMessageHistoryHoleOperation(.insert(clippedRange), peerId: peerId, threadId: threadId, namespace: namespace, space: .customTag(tagValue, MessageTags(rawValue: regularTag)), to: &operations)
+    }
+    
+    func remove(peerId: PeerId, threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let mappedTag = self.tagIdTable.get(tag: tag)
+        
+        self.ensureInitialized(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, regularTag: regularTag)
+        
+        self.removeInternal(peerId: peerId, threadId: threadId, tag: mappedTag, tagValue: tag, regularTag: regularTag, namespace: namespace, range: range, operations: &operations)
+    }
+    
+    private func removeInternal(peerId: PeerId, threadId: Int64?, tag: Int32, tagValue: MemoryBuffer, regularTag: UInt32, namespace: MessageId.Namespace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keyThreadId, keyTag, keyRegularTag) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keyTag == tag)
+            assert(keyRegularTag == regularTag)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if range.lowerBound <= holeRange.lowerBound && range.upperBound >= holeRange.upperBound {
+                removeKeys.append(upperId.id)
+            } else if range.overlaps(holeRange) {
+                removeKeys.append(upperId.id)
+                var holeIndices = IndexSet(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+                holeIndices.remove(integersIn: Int(range.lowerBound) ... Int(range.upperBound))
+                insertRanges.formUnion(holeIndices)
+            }
+        }
+        
+        let lowerScanBound = max(0, range.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), threadId: threadId, tag: tag, regularTag: regularTag), end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: tag, regularTag: regularTag).successor, values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), threadId: threadId, tag: tag, regularTag: regularTag), end: self.upperBound(peerId: peerId, namespace: namespace, threadId: threadId, tag: tag, regularTag: regularTag), values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 1)
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: id), threadId: threadId, tag: tag, regularTag: regularTag), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), threadId: threadId, tag: tag, regularTag: regularTag), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        if !removeKeys.isEmpty {
+            addMessageHistoryHoleOperation(.remove(range), peerId: peerId, threadId: threadId, namespace: namespace, space: .customTag(tagValue, MessageTags(rawValue: regularTag)), to: &operations)
+        }
+    }
+    
+    func resetAll() {
+        self.clearMemoryCache()
+        
+        self.valueBox.removeAllFromTable(self.table)
+        self.metadataTable.removePeerCustomTagInitializedList()
+    }
+}

submodules/Postbox/Sources/MessageCustomTagWithTagTable.swift

@@ -0,0 +1,179 @@
+import Foundation
+
+private func extractKey(_ key: ValueBoxKey) -> MessageIndex {
+    return MessageIndex(
+        id: MessageId(
+            peerId: PeerId(key.getInt64(0)),
+            namespace: key.getInt32(8 + 8 + 4 + 4),
+            id: key.getInt32(8 + 8 + 4 + 4 + 4 + 4)
+        ),
+        timestamp: key.getInt32(8 + 8 + 4 + 4 + 4)
+    )
+}
+
+class MessageCustomTagWithTagTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let messageCustomTagIdTable: MessageCustomTagIdTable
+    private let summaryTable: MessageHistoryTagsSummaryTable
+    private let summaryTags: MessageTags
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4 + 4)
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, messageCustomTagIdTable: MessageCustomTagIdTable, seedConfiguration: SeedConfiguration, summaryTable: MessageHistoryTagsSummaryTable) {
+        self.messageCustomTagIdTable = messageCustomTagIdTable
+        self.summaryTable = summaryTable
+        self.summaryTags = seedConfiguration.messageTagsWithSummary
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(threadId: Int64?, tag: Int32, regularTag: UInt32, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4 + 4)) -> ValueBoxKey {
+        key.setInt64(0, value: index.id.peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setUInt32(8 + 8 + 4, value: regularTag)
+        key.setInt32(8 + 8 + 4 + 4, value: index.id.namespace)
+        key.setInt32(8 + 8 + 4 + 4 + 4, value: index.timestamp)
+        key.setInt32(8 + 8 + 4 + 4 + 4 + 4, value: index.id.id)
+        return key
+    }
+    
+    private func lowerBound(threadId: Int64?, tag: Int32, regularTag: UInt32, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId ?? 0)
+        key.setInt32(8 + 8, value: tag)
+        key.setUInt32(8 + 8 + 4, value: regularTag)
+        key.setInt32(8 + 8 + 4 + 4, value: namespace)
+        return key
+    }
+    
+    private func upperBound(threadId: Int64?, tag: Int32, regularTag: UInt32, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        return self.lowerBound(threadId: threadId, tag: tag, regularTag: regularTag, peerId: peerId, namespace: namespace).successor
+    }
+    
+    func add(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, index: MessageIndex, isNewlyAdded: Bool, updatedSummaries: inout[MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        self.valueBox.set(self.table, key: self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index, key: self.sharedKey), value: MemoryBuffer())
+        
+        if self.summaryTags.contains(MessageTags(rawValue: regularTag)) {
+            self.summaryTable.addMessage(key: MessageHistoryTagsSummaryKey(tag: MessageTags(rawValue: regularTag), peerId: index.id.peerId, threadId: threadId, namespace: index.id.namespace, customTag: tag), id: index.id.id, isNewlyAdded: isNewlyAdded, updatedSummaries: &updatedSummaries, invalidateSummaries: &invalidateSummaries)
+        }
+    }
+    
+    func remove(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, index: MessageIndex, updatedSummaries: inout[MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        self.valueBox.remove(self.table, key: self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index, key: self.sharedKey), secure: false)
+        
+        if self.summaryTags.contains(MessageTags(rawValue: regularTag)) {
+            self.summaryTable.removeMessage(key: MessageHistoryTagsSummaryKey(tag: MessageTags(rawValue: regularTag), peerId: index.id.peerId, threadId: threadId, namespace: index.id.namespace, customTag: tag), id: index.id.id, updatedSummaries: &updatedSummaries, invalidateSummaries: &invalidateSummaries)
+        }
+    }
+    
+    func entryExists(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, index: MessageIndex) -> Bool {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        return self.valueBox.exists(self.table, key: self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index, key: self.sharedKey))
+    }
+    
+    func entryLocation(threadId: Int64?, index: MessageIndex, tag: MemoryBuffer, regularTag: UInt32) -> MessageHistoryEntryLocation? {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        if let _ = self.valueBox.get(self.table, key: self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index)) {
+            var greaterCount = 0
+            self.valueBox.range(self.table, start: self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index), end: self.upperBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: index.id.peerId, namespace: index.id.namespace), keys: { _ in
+                greaterCount += 1
+                return true
+            }, limit: 0)
+            
+            var lowerCount = 0
+            self.valueBox.range(self.table, start: self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index), end: self.lowerBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: index.id.peerId, namespace: index.id.namespace), keys: { _ in
+                lowerCount += 1
+                return true
+            }, limit: 0)
+            
+            return MessageHistoryEntryLocation(index: lowerCount, count: greaterCount + lowerCount + 1)
+        }
+        return nil
+    }
+    
+    func earlierIndices(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, minIndex: MessageIndex? = nil, count: Int) -> [MessageIndex] {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index).successor
+            } else {
+                key = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index)
+            }
+        } else {
+            key = self.upperBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: peerId, namespace: namespace)
+        }
+        let endKey: ValueBoxKey
+        if let minIndex = minIndex {
+            endKey = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: minIndex)
+        } else {
+            endKey = self.lowerBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: endKey, keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func laterIndices(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, count: Int) -> [MessageIndex] {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index).predecessor
+            } else {
+                key = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: index)
+            }
+        } else {
+            key = self.lowerBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: self.upperBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func getMessageCountInRange(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, peerId: PeerId, namespace: MessageId.Namespace, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        precondition(lowerBound.id.namespace == namespace)
+        precondition(upperBound.id.namespace == namespace)
+        var lowerBoundKey = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: lowerBound)
+        if lowerBound.timestamp > 1 {
+            lowerBoundKey = lowerBoundKey.predecessor
+        }
+        var upperBoundKey = self.key(threadId: threadId, tag: mappedTag, regularTag: regularTag, index: upperBound)
+        if upperBound.timestamp < Int32.max - 1 {
+            upperBoundKey = upperBoundKey.successor
+        }
+        return Int(self.valueBox.count(self.table, start: lowerBoundKey, end: upperBoundKey))
+    }
+    
+    func latestIndex(threadId: Int64?, tag: MemoryBuffer, regularTag: UInt32, peerId: PeerId, namespace: MessageId.Namespace) -> MessageIndex? {
+        let mappedTag = self.messageCustomTagIdTable.get(tag: tag)
+        
+        var result: MessageIndex?
+        self.valueBox.range(self.table, start: self.lowerBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: peerId, namespace: namespace), end: self.upperBound(threadId: threadId, tag: mappedTag, regularTag: regularTag, peerId: peerId, namespace: namespace), keys: { key in
+            result = extractKey(key)
+            return true
+        }, limit: 1)
+        return result
+    }
+}

submodules/Postbox/Sources/MessageGloballyUniqueIdTable.swift

@@ -0,0 +1,44 @@
+import Foundation
+
+final class MessageGloballyUniqueIdTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 8)
+    private let sharedBuffer = WriteBuffer()
+    
+    private func key(peerId: PeerId, id: Int64) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: peerId.toInt64())
+        self.sharedKey.setInt64(8, value: id)
+        return self.sharedKey
+    }
+    
+    func set(peerId: PeerId, globallyUniqueId: Int64, id: MessageId) {
+        self.sharedBuffer.reset()
+        var idPeerId: Int64 = id.peerId.toInt64()
+        var idNamespace: Int32 = id.namespace
+        var idId: Int32 = id.id
+        self.sharedBuffer.write(&idPeerId, offset: 0, length: 8)
+        self.sharedBuffer.write(&idNamespace, offset: 0, length: 4)
+        self.sharedBuffer.write(&idId, offset: 0, length: 4)
+        self.valueBox.set(self.table, key: self.key(peerId: peerId, id: globallyUniqueId), value: self.sharedBuffer)
+    }
+    
+    func get(peerId: PeerId, globallyUniqueId: Int64) -> MessageId? {
+        if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, id: globallyUniqueId)) {
+            var idPeerId: Int64 = 0
+            var idNamespace: Int32 = 0
+            var idId: Int32 = 0
+            value.read(&idPeerId, offset: 0, length: 8)
+            value.read(&idNamespace, offset: 0, length: 4)
+            value.read(&idId, offset: 0, length: 4)
+            return MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId)
+        }
+        return nil
+    }
+    
+    func remove(peerId: PeerId, globallyUniqueId: Int64) {
+        self.valueBox.remove(self.table, key: self.key(peerId: peerId, id: globallyUniqueId), secure: false)
+    }
+}

submodules/Postbox/Sources/MessageGroupView.swift

@@ -0,0 +1,65 @@
+import Foundation
+
+final class MutableMessageGroupView: MutablePostboxView {
+    fileprivate let id: MessageId
+    fileprivate let groupingKey: Int64?
+    fileprivate var messages: [Message] = []
+    
+    init(postbox: PostboxImpl, id: MessageId) {
+        self.id = id
+        self.messages = postbox.getMessageGroup(at: id) ?? []
+        self.groupingKey = self.messages.first?.groupingKey
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        if let operations = transaction.currentOperationsByPeerId[self.id.peerId] {
+            outer: for operation in operations {
+                switch operation {
+                case let .InsertMessage(message):
+                    if let groupingKey = self.groupingKey, message.groupingKey == groupingKey {
+                        updated = true
+                        break outer
+                    } else if message.id == self.id {
+                        updated = true
+                        break outer
+                    }
+                case let .Remove(indices):
+                    for index in indices {
+                        for message in self.messages {
+                            if index.0.id == message.id {
+                                updated = true
+                                break outer
+                            }
+                        }
+                    }
+                default:
+                    break
+                }
+            }
+        }
+        if updated {
+            self.messages = postbox.getMessageGroup(at: self.id) ?? []
+            
+            return true
+        } else {
+            return false
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageGroupView(self)
+    }
+}
+
+public final class MessageGroupView: PostboxView {
+    public let messages: [Message]
+    
+    init(_ view: MutableMessageGroupView) {
+        self.messages = view.messages
+    }
+}

submodules/Postbox/Sources/MessageHistoryAnchorIndex.swift

@@ -0,0 +1,51 @@
+import Foundation
+
+public enum MessageHistoryAnchorIndex: Comparable {
+    case message(MessageIndex)
+    case lowerBound
+    case upperBound
+    
+    public static func <(lhs: MessageHistoryAnchorIndex, rhs: MessageHistoryAnchorIndex) -> Bool {
+        switch lhs {
+            case let .message(lhsIndex):
+                switch rhs {
+                    case let .message(rhsIndex):
+                        return lhsIndex < rhsIndex
+                    case .lowerBound:
+                        return false
+                    case .upperBound:
+                        return true
+                }
+            case .lowerBound:
+                if case .lowerBound = rhs {
+                    return false
+                } else {
+                    return true
+                }
+            case .upperBound:
+                return false
+        }
+    }
+    
+    public func isLess(than: MessageIndex) -> Bool {
+        switch self {
+            case .lowerBound:
+                return true
+            case .upperBound:
+                return false
+            case let .message(index):
+                return index < than
+        }
+    }
+    
+    public func isLessOrEqual(to: MessageIndex) -> Bool {
+        switch self {
+            case .lowerBound:
+                return true
+            case .upperBound:
+                return false
+            case let .message(index):
+                return index <= to
+        }
+    }
+}

submodules/Postbox/Sources/MessageHistoryFailedTable.swift

@@ -0,0 +1,74 @@
+import Foundation
+
+final class MessageHistoryFailedTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 4 + 4)
+    
+    private(set) var updatedPeerIds = Set<PeerId>()
+    private(set) var updatedMessageIds = Set<MessageId>()
+
+    private func key(_ id: MessageId) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: id.peerId.toInt64())
+        self.sharedKey.setInt32(8, value: id.namespace)
+        self.sharedKey.setInt32(8 + 4, value: id.id)
+        
+        return self.sharedKey
+    }
+    
+    private func lowerBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key.successor
+    }
+    
+    func add(_ id: MessageId) {
+        self.valueBox.set(self.table, key: self.key(id), value: MemoryBuffer())
+        self.updatedPeerIds.insert(id.peerId)
+        self.updatedMessageIds.insert(id)
+    }
+    
+    func remove(_ id: MessageId) {
+        self.valueBox.remove(self.table, key: self.key(id), secure: false)
+        self.updatedPeerIds.insert(id.peerId)
+        self.updatedMessageIds.remove(id)
+    }
+    
+    func get(peerId: PeerId) -> [MessageId] {
+        var ids:[MessageId] = []
+        self.valueBox.range(self.table, start: self.lowerBound(peerId: peerId), end: self.upperBound(peerId: peerId), keys: { key in
+            
+            let peerId = PeerId(key.getInt64(0))
+            let namespace = key.getInt32(8)
+            let id = key.getInt32(8 + 4)
+            ids.append(MessageId(peerId: peerId, namespace: namespace, id: id))
+            
+            return false
+        }, limit: 100)
+        
+        self.updatedMessageIds = self.updatedMessageIds.union(ids)
+        
+        return ids
+    }
+    
+    func contains(peerId: PeerId) -> Bool {
+        var result = false
+        self.valueBox.range(self.table, start: self.lowerBound(peerId: peerId), end: self.upperBound(peerId: peerId), keys: { _ in
+            result = true
+            return false
+        }, limit: 1)
+        return result
+    }
+    
+    override func beforeCommit() {
+        self.updatedPeerIds.removeAll()
+    }
+}

submodules/Postbox/Sources/MessageHistoryHoleIndexTable.swift

@@ -0,0 +1,491 @@
+import Foundation
+
+struct MessageHistoryIndexHoleOperationKey: Hashable {
+    let peerId: PeerId
+    let namespace: MessageId.Namespace
+    let threadId: Int64?
+    let space: MessageHistoryHoleOperationSpace
+    
+    init(peerId: PeerId, namespace: MessageId.Namespace, threadId: Int64?, space: MessageHistoryHoleOperationSpace) {
+        self.peerId = peerId
+        self.namespace = namespace
+        self.threadId = threadId
+        self.space = space
+    }
+}
+
+enum MessageHistoryIndexHoleOperation {
+    case insert(ClosedRange<MessageId.Id>)
+    case remove(ClosedRange<MessageId.Id>)
+}
+
+public enum MessageHistoryHoleSpace: Equatable, Hashable, CustomStringConvertible {
+    case everywhere
+    case tag(MessageTags)
+    
+    public var description: String {
+        switch self {
+        case .everywhere:
+            return ".everywhere"
+        case let .tag(tags):
+            return ".tag\(tags.rawValue)"
+        }
+    }
+}
+
+public enum MessageHistoryHoleOperationSpace: Equatable, Hashable, CustomStringConvertible {
+    case everywhere
+    case tag(MessageTags)
+    case customTag(MemoryBuffer, MessageTags?)
+    
+    public var description: String {
+        switch self {
+        case .everywhere:
+            return ".everywhere"
+        case let .tag(tags):
+            return ".tag\(tags.rawValue)"
+        case let .customTag(customTag, regularTag):
+            if let regularTag {
+                return ".customTag\(customTag)_tag\(regularTag.rawValue)"
+            } else {
+                return ".customTag\(customTag)"
+            }
+        }
+    }
+}
+
+public extension MessageHistoryHoleOperationSpace {
+    init(_ space: MessageHistoryHoleSpace) {
+        switch space {
+        case .everywhere:
+            self = .everywhere
+        case let .tag(tag):
+            self = .tag(tag)
+        }
+    }
+}
+
+func addMessageHistoryHoleOperation(_ operation: MessageHistoryIndexHoleOperation, peerId: PeerId, threadId: Int64?, namespace: MessageId.Namespace, space: MessageHistoryHoleOperationSpace, to operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+    let key = MessageHistoryIndexHoleOperationKey(peerId: peerId, namespace: namespace, threadId: threadId, space: space)
+    if operations[key] == nil {
+        operations[key] = []
+    }
+    operations[key]!.append(operation)
+}
+
+private func decomposeKey(_ key: ValueBoxKey) -> (id: MessageId, space: MessageHistoryHoleSpace) {
+    let tag = MessageTags(rawValue: key.getUInt32(8 + 4))
+    let space: MessageHistoryHoleSpace
+    if tag.rawValue == 0 {
+        space = .everywhere
+    } else {
+        space = .tag(tag)
+    }
+    return (MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8), id: key.getInt32(8 + 4 + 4)), space)
+}
+
+private func decodeValue(value: ReadBuffer, peerId: PeerId, namespace: MessageId.Namespace) -> MessageId {
+    var id: Int32 = 0
+    value.read(&id, offset: 0, length: 4)
+    return MessageId(peerId: peerId, namespace: namespace, id: id)
+}
+
+final class MessageHistoryHoleIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    let metadataTable: MessageHistoryMetadataTable
+    let seedConfiguration: SeedConfiguration
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) {
+        self.seedConfiguration = seedConfiguration
+        self.metadataTable = metadataTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(id: MessageId, space: MessageHistoryHoleSpace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4 + 4 + 4)
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt32(8, value: id.namespace)
+        let tagValue: UInt32
+        switch space {
+            case .everywhere:
+                tagValue = 0
+            case let .tag(tag):
+                tagValue = tag.rawValue
+        }
+        key.setUInt32(8 + 4, value: tagValue)
+        key.setInt32(8 + 4 + 4, value: id.id)
+        return key
+    }
+    
+    private func lowerBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId).successor
+    }
+    
+    private func lowerBound(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        let tagValue: UInt32
+        switch space {
+            case .everywhere:
+                tagValue = 0
+            case let .tag(tag):
+                tagValue = tag.rawValue
+        }
+        key.setUInt32(8 + 4, value: tagValue)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        let tagValue: UInt32
+        switch space {
+            case .everywhere:
+                tagValue = 0
+            case let .tag(tag):
+                tagValue = tag.rawValue
+        }
+        key.setUInt32(8 + 4, value: tagValue)
+        return key.successor
+    }
+    
+    private func namespaceLowerBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        return key
+    }
+    
+    private func namespaceUpperBound(peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        return key.successor
+    }
+    
+    private func ensureInitialized(peerId: PeerId) {
+        if !self.metadataTable.isInitialized(peerId) {
+            self.metadataTable.setInitialized(peerId)
+            if let tagsByNamespace = self.seedConfiguration.messageHoles[peerId.namespace] {
+                for (namespace, tags) in tagsByNamespace {
+                    for tag in tags {
+                        self.metadataTable.setPeerTagInitialized(peerId: peerId, tag: tag)
+                    }
+                    var operations: [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]] = [:]
+                    self.add(peerId: peerId, namespace: namespace, space: .everywhere, range: 1 ... (Int32.max - 1), operations: &operations)
+                }
+            }
+        } else {
+            if let tagsByNamespace = self.seedConfiguration.upgradedMessageHoles[peerId.namespace] {
+                for (namespace, tags) in tagsByNamespace {
+                    for tag in tags {
+                        if !self.metadataTable.isPeerTagInitialized(peerId: peerId, tag: tag) {
+                            self.metadataTable.setPeerTagInitialized(peerId: peerId, tag: tag)
+                            var operations: [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]] = [:]
+                            self.add(peerId: peerId, namespace: namespace, space: .tag(tag), range: 1 ... (Int32.max - 1), operations: &operations)
+                        }
+                    }
+                }
+            }
+        }
+    }
+    
+    func existingNamespaces(peerId: PeerId, holeSpace: MessageHistoryHoleSpace) -> Set<MessageId.Namespace> {
+        self.ensureInitialized(peerId: peerId)
+        
+        var result = Set<MessageId.Namespace>()
+        var currentLowerBound = self.lowerBound(peerId: peerId)
+        let upperBound = self.upperBound(peerId: peerId)
+        while true {
+            var idAndSpace: (MessageId, MessageHistoryHoleSpace)?
+            self.valueBox.range(self.table, start: currentLowerBound, end: upperBound, keys: { key in
+                idAndSpace = decomposeKey(key)
+                return false
+            }, limit: 1)
+            if let (id, space) = idAndSpace {
+                if space == holeSpace {
+                    result.insert(id.namespace)
+                }
+                currentLowerBound = self.upperBound(peerId: peerId, namespace: id.namespace, space: space)
+            } else {
+                break
+            }
+        }
+        return result
+    }
+    
+    private func scanSpaces(peerId: PeerId, namespace: MessageId.Namespace) -> [MessageHistoryHoleSpace] {
+        self.ensureInitialized(peerId: peerId)
+        
+        var currentLowerBound = self.namespaceLowerBound(peerId: peerId, namespace: namespace)
+        var result: [MessageHistoryHoleSpace] = []
+        while true {
+            var found = false
+            self.valueBox.range(self.table, start: currentLowerBound, end: self.namespaceUpperBound(peerId: peerId, namespace: namespace), keys: { key in
+                let space = decomposeKey(key).space
+                result.append(space)
+                currentLowerBound = self.upperBound(peerId: peerId, namespace: namespace, space: space)
+                found = true
+                return false
+            }, limit: 1)
+            if !found {
+                break
+            }
+        }
+        assert(Set(result).count == result.count)
+        return result
+    }
+    
+    func containing(id: MessageId) -> [MessageHistoryHoleSpace: ClosedRange<MessageId.Id>] {
+        self.ensureInitialized(peerId: id.peerId)
+        
+        var result: [MessageHistoryHoleSpace: ClosedRange<MessageId.Id>] = [:]
+        for space in self.scanSpaces(peerId: id.peerId, namespace: id.namespace) {
+            self.valueBox.range(self.table, start: self.key(id: id, space: space), end: self.upperBound(peerId: id.peerId, namespace: id.namespace, space: space), values: { key, value in
+                let (upperId, keySpace) = decomposeKey(key)
+                assert(keySpace == space)
+                assert(upperId.peerId == id.peerId)
+                assert(upperId.namespace == id.namespace)
+                let lowerId = decodeValue(value: value, peerId: id.peerId, namespace: id.namespace)
+                let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+                result[space] = holeRange
+                return false
+            }, limit: 1)
+        }
+        return result
+    }
+    
+    func closest(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>) -> IndexSet {
+        self.ensureInitialized(peerId: peerId)
+        
+        var result = IndexSet()
+        
+        func processIntersectingRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keySpace) = decomposeKey(key)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if holeRange.overlaps(range) {
+                result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+            }
+        }
+        
+        func processEdgeRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keySpace) = decomposeKey(key)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+        }
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), space: space).predecessor, end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space).successor, values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space), end: self.upperBound(peerId: peerId, namespace: namespace, space: space), values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 1)
+        
+        if !result.contains(Int(range.lowerBound)) {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), space: space), end: self.lowerBound(peerId: peerId, namespace: namespace, space: space), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        if !result.contains(Int(range.upperBound)) {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space), end: self.upperBound(peerId: peerId, namespace: namespace, space: space), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        
+        return result
+    }
+    
+    func add(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        self.ensureInitialized(peerId: peerId)
+        
+        self.addInternal(peerId: peerId, namespace: namespace, space: space, range: range, operations: &operations)
+        
+        switch space {
+            case .everywhere:
+                if let namespaceHoleTags = self.seedConfiguration.messageHoles[peerId.namespace]?[namespace] {
+                    for tag in namespaceHoleTags {
+                        self.addInternal(peerId: peerId, namespace: namespace, space: .tag(tag), range: range, operations: &operations)
+                    }
+                }
+            case .tag:
+                break
+        }
+    }
+    
+    private func addInternal(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let clippedLowerBound = max(1, range.lowerBound)
+        let clippedUpperBound = min(Int32.max - 1, range.upperBound)
+        if clippedLowerBound > clippedUpperBound {
+            return
+        }
+        let clippedRange = clippedLowerBound ... clippedUpperBound
+        
+        var insertedIndices = IndexSet()
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        var alreadyMapped = false
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keySpace) = decomposeKey(key)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<Int32> = lowerId.id ... upperId.id
+            if clippedRange.lowerBound >= holeRange.lowerBound && clippedRange.upperBound <= holeRange.upperBound {
+                alreadyMapped = true
+                return
+            } else if clippedRange.overlaps(holeRange) || (holeRange.upperBound != Int32.max && clippedRange.lowerBound == holeRange.upperBound + 1) || clippedRange.upperBound == holeRange.lowerBound - 1 {
+                removeKeys.append(upperId.id)
+                let unionRange: ClosedRange = min(clippedRange.lowerBound, holeRange.lowerBound) ... max(clippedRange.upperBound, holeRange.upperBound)
+                insertRanges.insert(integersIn: Int(unionRange.lowerBound) ... Int(unionRange.upperBound))
+            }
+        }
+        
+        let lowerScanBound = max(0, clippedRange.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), space: space), end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), space: space).successor, values: { key, value in
+            processRange(key, value)
+            if alreadyMapped {
+                return false
+            }
+            return true
+        }, limit: 0)
+        
+        if !alreadyMapped {
+            self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), space: space), end: self.upperBound(peerId: peerId, namespace: namespace, space: space), values: { key, value in
+                processRange(key, value)
+                if alreadyMapped {
+                    return false
+                }
+                return true
+            }, limit: 1)
+        }
+        
+        if alreadyMapped {
+            return
+        }
+        
+        insertRanges.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        insertedIndices.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: id), space: space), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), space: space), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        addMessageHistoryHoleOperation(.insert(clippedRange), peerId: peerId, threadId: nil, namespace: namespace, space: MessageHistoryHoleOperationSpace(space), to: &operations)
+    }
+    
+    func remove(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        self.ensureInitialized(peerId: peerId)
+        
+        self.removeInternal(peerId: peerId, namespace: namespace, space: space, range: range, operations: &operations)
+        
+        switch space {
+            case .everywhere:
+                if let namespaceHoleTags = self.seedConfiguration.messageHoles[peerId.namespace]?[namespace] {
+                    for tag in namespaceHoleTags {
+                        self.removeInternal(peerId: peerId, namespace: namespace, space: .tag(tag), range: range, operations: &operations)
+                    }
+                }
+            case .tag:
+                break
+        }
+    }
+    
+    private func removeInternal(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (upperId, keySpace) = decomposeKey(key)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if range.lowerBound <= holeRange.lowerBound && range.upperBound >= holeRange.upperBound {
+                removeKeys.append(upperId.id)
+            } else if range.overlaps(holeRange) {
+                removeKeys.append(upperId.id)
+                var holeIndices = IndexSet(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+                holeIndices.remove(integersIn: Int(range.lowerBound) ... Int(range.upperBound))
+                insertRanges.formUnion(holeIndices)
+            }
+        }
+        
+        let lowerScanBound = max(0, range.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), space: space), end: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space).successor, values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space), end: self.upperBound(peerId: peerId, namespace: namespace, space: space), values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 1)
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: id), space: space), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), space: space), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        if !removeKeys.isEmpty {
+            addMessageHistoryHoleOperation(.remove(range), peerId: peerId, threadId: nil, namespace: namespace, space: MessageHistoryHoleOperationSpace(space), to: &operations)
+            postboxLog("MessageHistoryHoleIndexTable: removeInternal peerId: \(peerId) namespace: \(namespace) space: \(space) range: \(range)")
+        }
+    }
+    
+    func debugList(peerId: PeerId, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> [ClosedRange<MessageId.Id>] {
+        var result: [ClosedRange<MessageId.Id>] = []
+        self.valueBox.range(self.table, start: self.lowerBound(peerId: peerId, namespace: namespace, space: space), end: self.upperBound(peerId: peerId, namespace: namespace, space: space), values: { key, value in
+            let (upperId, keySpace) = decomposeKey(key)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            result.append(holeRange)
+            return true
+        }, limit: 0)
+        return result
+    }
+}

submodules/Postbox/Sources/MessageHistoryHolesView.swift

@@ -0,0 +1,82 @@
+import Foundation
+
+public struct MessageHistoryHolesViewEntry: Equatable, Hashable, CustomStringConvertible {
+    public let hole: MessageHistoryViewHole
+    public let direction: MessageHistoryViewRelativeHoleDirection
+    public let space: MessageHistoryHoleOperationSpace
+    public let count: Int
+    public let userId: Int64?
+    
+    public init(hole: MessageHistoryViewHole, direction: MessageHistoryViewRelativeHoleDirection, space: MessageHistoryHoleOperationSpace, count: Int, userId: Int64?) {
+        self.hole = hole
+        self.direction = direction
+        self.space = space
+        self.count = count
+        self.userId = userId
+    }
+    
+    public var description: String {
+        return "hole: \(self.hole), direction: \(self.direction), space: \(self.space), count: \(self.count), userId: \(String(describing: self.userId))"
+    }
+}
+
+final class MutableMessageHistoryHolesView {
+    fileprivate var entries = Set<MessageHistoryHolesViewEntry>()
+    
+    init() {
+    }
+    
+    func update(_ holes: Set<MessageHistoryHolesViewEntry>) -> Bool {
+        if self.entries != holes {
+            self.entries = holes
+            return true
+        } else {
+            return false
+        }
+    }
+}
+
+public final class MessageHistoryHolesView {
+    public let entries: Set<MessageHistoryHolesViewEntry>
+    
+    init(_ mutableView: MutableMessageHistoryHolesView) {
+        self.entries = mutableView.entries
+    }
+}
+
+public struct MessageHistoryExternalHolesViewEntry: Equatable, Hashable {
+    public let hole: MessageHistoryViewHole
+    public let direction: MessageHistoryViewRelativeHoleDirection
+    public let count: Int
+    
+    public init(hole: MessageHistoryViewHole, direction: MessageHistoryViewRelativeHoleDirection, count: Int) {
+        self.hole = hole
+        self.direction = direction
+        self.count = count
+    }
+}
+
+final class MutableMessageHistoryExternalHolesView {
+    fileprivate var entries = Set<MessageHistoryExternalHolesViewEntry>()
+    
+    init() {
+    }
+    
+    func update(_ holes: Set<MessageHistoryExternalHolesViewEntry>) -> Bool {
+        if self.entries != holes {
+            self.entries = holes
+            return true
+        } else {
+            return false
+        }
+    }
+}
+
+public final class MessageHistoryExternalHolesView {
+    public let entries: Set<MessageHistoryExternalHolesViewEntry>
+    
+    init(_ mutableView: MutableMessageHistoryExternalHolesView) {
+        self.entries = mutableView.entries
+    }
+}
+

submodules/Postbox/Sources/MessageHistoryIndexTable.swift

@@ -0,0 +1,347 @@
+import Foundation
+
+public enum AddMessagesLocation {
+    case Random
+    case UpperHistoryBlock
+}
+
+enum MessageHistoryIndexOperation {
+    case InsertMessage(InternalStoreMessage)
+    case InsertExistingMessage(InternalStoreMessage)
+    case Remove(index: MessageIndex)
+    case Update(MessageIndex, InternalStoreMessage)
+    case UpdateTimestamp(MessageIndex, Int32)
+}
+
+private let HistoryEntryTypeMask: Int8 = 1
+private let HistoryEntryTypeMessage: Int8 = 0
+private let HistoryEntryMessageFlagIncoming: Int8 = 1 << 1
+
+private func readHistoryIndexEntry(_ peerId: PeerId, namespace: MessageId.Namespace, key: ValueBoxKey, value: ReadBuffer) -> MessageIndex {
+    var flags: Int8 = 0
+    value.read(&flags, offset: 0, length: 1)
+    var timestamp: Int32 = 0
+    value.read(&timestamp, offset: 0, length: 4)
+    let index = MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: key.getInt32(8 + 4)), timestamp: timestamp)
+    return index
+}
+
+private func modifyHistoryIndexEntryTimestamp(value: ReadBuffer, timestamp: Int32) -> MemoryBuffer {
+    let buffer = WriteBuffer()
+    buffer.write(value.memory.advanced(by: 0), offset: 0, length: 1)
+    var varTimestamp: Int32 = timestamp
+    buffer.write(&varTimestamp, offset: 0, length: 4)
+    buffer.write(value.memory.advanced(by: 5), offset: 0, length: value.length - 5)
+    return buffer
+}
+
+final class MessageHistoryIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let messageHistoryHoleIndexTable: MessageHistoryHoleIndexTable
+    private let globalMessageIdsTable: GlobalMessageIdsTable
+    private let metadataTable: MessageHistoryMetadataTable
+    private let seedConfiguration: SeedConfiguration
+    
+    private var cachedExistingNamespaces: [PeerId: Set<MessageId.Namespace>] = [:]
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, messageHistoryHoleIndexTable: MessageHistoryHoleIndexTable, globalMessageIdsTable: GlobalMessageIdsTable, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) {
+        self.messageHistoryHoleIndexTable = messageHistoryHoleIndexTable
+        self.globalMessageIdsTable = globalMessageIdsTable
+        self.seedConfiguration = seedConfiguration
+        self.metadataTable = metadataTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(_ id: MessageId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4 + 4)
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt32(8, value: id.namespace)
+        key.setInt32(8 + 4, value: id.id)
+        return key
+    }
+    
+    private func lowerBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key
+    }
+    
+    private func lowerBound(_ peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        return key
+    }
+    
+    private func upperBound(_ peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: namespace)
+        return key.successor
+    }
+    
+    private func upperBound(peerId: PeerId) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId).successor
+    }
+    
+    func addMessages(_ messages: [InternalStoreMessage], operations: inout [MessageHistoryIndexOperation]) {
+        if messages.count == 0 {
+            return
+        }
+        
+        for message in messages {
+            let index = MessageIndex(id: message.id, timestamp: message.timestamp)
+            
+            if let currentIndex = self.getIndex(index.id) {
+                if currentIndex.timestamp == index.timestamp {
+                    operations.append(.InsertExistingMessage(message))
+                } else {
+                    self.justRemove(currentIndex, operations: &operations)
+                    self.justInsertMessage(message, operations: &operations)
+                }
+            } else {
+                self.justInsertMessage(message, operations: &operations)
+                
+                self.cachedExistingNamespaces[message.id.peerId]?.insert(message.id.namespace)
+            }
+        }
+    }
+    
+    func removeMessage(_ id: MessageId, operations: inout [MessageHistoryIndexOperation]) {
+        if let index = self.getIndex(id) {
+            self.justRemove(index, operations: &operations)
+        }
+    }
+    
+    func removeMessagesInRange(peerId: PeerId, namespace: MessageId.Namespace, minId: MessageId.Id, maxId: MessageId.Id, operations: inout [MessageHistoryIndexOperation]) {
+        if minId > maxId {
+            assertionFailure()
+            return
+        }
+        var removeMessageIds: [MessageId] = []
+        self.valueBox.range(self.table, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: minId)).predecessor, end: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)).successor, values: { key, value in
+            let index = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value)
+            removeMessageIds.append(index.id)
+            return true
+        }, limit: 0)
+        for id in removeMessageIds {
+            self.removeMessage(id, operations: &operations)
+        }
+    }
+    
+    func updateMessage(_ id: MessageId, message: InternalStoreMessage, operations: inout [MessageHistoryIndexOperation]) {
+        if let previousIndex = self.getIndex(id) {
+            if previousIndex != message.index {
+                var intermediateOperations: [MessageHistoryIndexOperation] = []
+                self.removeMessage(id, operations: &intermediateOperations)
+                self.addMessages([message], operations: &intermediateOperations)
+                
+                for operation in intermediateOperations {
+                    switch operation {
+                        case let .Remove(index) where index == previousIndex:
+                            operations.append(.Update(previousIndex, message))
+                        case let .InsertMessage(insertMessage) where insertMessage.index == message.index:
+                            break
+                        case let .InsertExistingMessage(insertMessage) where insertMessage.index == message.index:
+                            operations.removeAll()
+                            operations.append(.Remove(index: previousIndex))
+                            operations.append(.Update(insertMessage.index, message))
+                        default:
+                            operations.append(operation)
+                    }
+                }
+            } else {
+                operations.append(.Update(previousIndex, message))
+            }
+        }
+    }
+    
+    func updateTimestamp(_ id: MessageId, timestamp: Int32, operations: inout [MessageHistoryIndexOperation]) {
+        if let previousData = self.valueBox.get(self.table, key: self.key(id)), let previousIndex = self.getIndex(id), previousIndex.timestamp != timestamp {
+            let updatedEntry = modifyHistoryIndexEntryTimestamp(value: previousData, timestamp: timestamp)
+            self.valueBox.remove(self.table, key: self.key(id), secure: false)
+            self.valueBox.set(self.table, key: self.key(id), value: updatedEntry)
+            
+            operations.append(.UpdateTimestamp(MessageIndex(id: id, timestamp: previousIndex.timestamp), timestamp))
+        }
+    }
+    
+    private func justInsertMessage(_ message: InternalStoreMessage, operations: inout [MessageHistoryIndexOperation]) {
+        let index = MessageIndex(id: message.id, timestamp: message.timestamp)
+        
+        let value = WriteBuffer()
+        var flags: Int8 = HistoryEntryTypeMessage
+        if !message.flags.intersection(.IsIncomingMask).isEmpty {
+            flags |= HistoryEntryMessageFlagIncoming
+        }
+        var timestamp: Int32 = index.timestamp
+        value.write(&flags, offset: 0, length: 1)
+        value.write(&timestamp, offset: 0, length: 4)
+        self.valueBox.set(self.table, key: self.key(index.id), value: value)
+        
+        operations.append(.InsertMessage(message))
+        
+        if self.seedConfiguration.globalMessageIdsPeerIdNamespaces.contains(GlobalMessageIdsNamespace(peerIdNamespace: index.id.peerId.namespace, messageIdNamespace: index.id.namespace)) {
+            self.globalMessageIdsTable.set(index.id.id, id: index.id)
+        }
+    }
+    
+    private func justRemove(_ index: MessageIndex, operations: inout [MessageHistoryIndexOperation]) {
+        self.valueBox.remove(self.table, key: self.key(index.id), secure: false)
+        
+        operations.append(.Remove(index: index))
+        if self.seedConfiguration.globalMessageIdsPeerIdNamespaces.contains(GlobalMessageIdsNamespace(peerIdNamespace: index.id.peerId.namespace, messageIdNamespace: index.id.namespace)) {
+            self.globalMessageIdsTable.remove(index.id.id)
+        }
+    }
+    
+    func getIndex(_ id: MessageId) -> MessageIndex? {
+        let key = self.key(id)
+        if let value = self.valueBox.get(self.table, key: key) {
+            return readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+        } else {
+            return nil
+        }
+    }
+    
+    func top(_ peerId: PeerId, namespace: MessageId.Namespace) -> MessageIndex? {
+        var index: MessageIndex?
+        self.valueBox.range(self.table, start: self.upperBound(peerId, namespace: namespace), end: self.lowerBound(peerId, namespace: namespace), values: { key, value in
+            index = readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value)
+            return false
+        }, limit: 1)
+        return index
+    }
+    
+    func exists(_ id: MessageId) -> Bool {
+        return self.valueBox.exists(self.table, key: self.key(id))
+    }
+    
+    func incomingMessageCountInRange(_ peerId: PeerId, namespace: MessageId.Namespace, minId: MessageId.Id, maxId: MessageId.Id) -> (Int, Bool) {
+        var count = 0
+        var holes = false
+        if minId <= maxId {
+            self.valueBox.range(self.table, start: self.key(MessageId(peerId: peerId, namespace: namespace, id: minId)).predecessor, end: self.key(MessageId(peerId: peerId, namespace: namespace, id: maxId)).successor, values: { _, value in
+                var flags: Int8 = 0
+                value.read(&flags, offset: 0, length: 1)
+                if (flags & HistoryEntryMessageFlagIncoming) != 0 {
+                    count += 1
+                }
+                return true
+            }, limit: 0)
+            
+            holes = !self.messageHistoryHoleIndexTable.closest(peerId: peerId, namespace: namespace, space: .everywhere, range: minId ... maxId).isEmpty
+        }
+        
+        return (count, holes)
+    }
+    
+    func incomingMessageCountInIds(_ peerId: PeerId, namespace: MessageId.Namespace, ids: [MessageId.Id]) -> (Int, Bool) {
+        var count = 0
+        var holes = false
+        
+        for id in ids {
+            if let value = self.valueBox.get(self.table, key: self.key(MessageId(peerId: peerId, namespace: namespace, id: id))) {
+                var flags: Int8 = 0
+                value.read(&flags, offset: 0, length: 1)
+                if (flags & HistoryEntryMessageFlagIncoming) != 0 {
+                    count += 1
+                }
+                if !self.messageHistoryHoleIndexTable.containing(id: MessageId(peerId: peerId, namespace: namespace, id: id)).isEmpty {
+                    holes = true
+                }
+            }
+        }
+        
+        return (count, holes)
+    }
+    
+    func indexForId(higherThan id: MessageId) -> MessageIndex? {
+        var result: MessageIndex?
+        self.valueBox.range(self.table, start: self.key(id), end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in
+            result = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+            return false
+        }, limit: 1)
+        return result
+    }
+    
+    func earlierEntries(id: MessageId, count: Int) -> [MessageIndex] {
+        var entries: [MessageIndex] = []
+        let key = self.key(id)
+        self.valueBox.range(self.table, start: key, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in
+            entries.append(readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value))
+            return true
+        }, limit: count)
+        return entries
+    }
+    
+    func existingNamespaces(peerId: PeerId) -> Set<MessageId.Namespace> {
+        if let cached = self.cachedExistingNamespaces[peerId] {
+            return cached
+        } else {
+            let namespaces = Set(self.fetchExistingNamespaces(peerId: peerId))
+            self.cachedExistingNamespaces[peerId] = namespaces
+            return namespaces
+        }
+    }
+    
+    private func fetchExistingNamespaces(peerId: PeerId) -> [MessageId.Namespace] {
+        var result: [MessageId.Namespace] = []
+        var lowerBound = self.lowerBound(peerId: peerId)
+        let upperBound = self.upperBound(peerId: peerId)
+        while true {
+            var namespace: MessageId.Namespace?
+            self.valueBox.range(self.table, start: lowerBound, end: upperBound, keys: { key in
+                assert(key.getInt64(0) == peerId.toInt64())
+                namespace = key.getInt32(8)
+                return false
+            }, limit: 1)
+            if let namespace = namespace {
+                result.append(namespace)
+                lowerBound = self.lowerBound(peerId, namespace: namespace + 1)
+            } else {
+                break
+            }
+        }
+        return result
+    }
+    
+    func debugList(_ peerId: PeerId, namespace: MessageId.Namespace) -> [MessageIndex] {
+        var list: [MessageIndex] = []
+        self.valueBox.range(self.table, start: self.lowerBound(peerId, namespace: namespace), end: self.upperBound(peerId, namespace: namespace), values: { key, value in
+            list.append(readHistoryIndexEntry(peerId, namespace: namespace, key: key, value: value))
+            
+            return true
+        }, limit: 0)
+        return list
+    }
+    
+    func closestIndex(id: MessageId) -> MessageIndex? {
+        if let index = self.getIndex(id) {
+            return index
+        } else {
+            var index: MessageIndex?
+            self.valueBox.range(self.table, start: self.key(id).successor, end: self.lowerBound(id.peerId, namespace: id.namespace), values: { key, value in
+                index = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+                return true
+            }, limit: 1)
+            
+            if index == nil {
+                self.valueBox.range(self.table, start: self.key(id).predecessor, end: self.upperBound(id.peerId, namespace: id.namespace), values: { key, value in
+                    index = readHistoryIndexEntry(id.peerId, namespace: id.namespace, key: key, value: value)
+                    return true
+                }, limit: 1)
+            }
+            
+            return index
+        }
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedExistingNamespaces.removeAll()
+    }
+}

submodules/Postbox/Sources/MessageHistoryMetadataTable.swift

@@ -0,0 +1,606 @@
+import Foundation
+
+private enum MetadataPrefix: Int8 {
+    case ChatListInitialized = 0
+    case PeerNextMessageIdByNamespace = 2
+    case NextStableMessageId = 3
+    case ChatListTotalUnreadState = 4
+    case NextPeerOperationLogIndex = 5
+    case ChatListGroupInitialized = 6
+    case GroupFeedIndexInitialized = 7
+    case ShouldReindexUnreadCounts = 8
+    case PeerHistoryInitialized = 9
+    case ShouldReindexUnreadCountsState = 10
+    case TotalUnreadCountStates = 11
+    case PeerHistoryTagInitialized = 12
+    case PeerHistoryThreadHoleIndexInitialized = 13
+    case NextCustomTagId = 14
+    case PeerHistoryCustomTagInitialized = 15
+    case PeerHistoryCustomTagWithTagInitialized = 16
+    case PeerHistoryCustomTagWithTagReindexed = 17
+}
+
+public struct ChatListTotalUnreadCounters: PostboxCoding, Equatable {
+    public var messageCount: Int32
+    public var chatCount: Int32
+    
+    public init(messageCount: Int32, chatCount: Int32) {
+        self.messageCount = messageCount
+        self.chatCount = chatCount
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.messageCount = decoder.decodeInt32ForKey("m", orElse: 0)
+        self.chatCount = decoder.decodeInt32ForKey("c", orElse: 0)
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeInt32(self.messageCount, forKey: "m")
+        encoder.encodeInt32(self.chatCount, forKey: "c")
+    }
+}
+
+private struct InitializedChatListKey: Hashable {
+    let groupId: PeerGroupId
+}
+
+final class MessageHistoryMetadataTable: Table {
+    private struct PeerIdThreadIdAndTag: Hashable {
+        var peerId: PeerId
+        var threadId: Int64?
+        var tag: Int32
+        var regularTag: UInt32?
+        
+        init(peerId: PeerId, threadId: Int64?, tag: Int32, regularTag: UInt32?) {
+            self.peerId = peerId
+            self.threadId = threadId
+            self.tag = tag
+            self.regularTag = regularTag
+        }
+    }
+    
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    let sharedPeerHistoryInitializedKey = ValueBoxKey(length: 8 + 1)
+    let sharedPeerThreadHoleIndexInitializedKey = ValueBoxKey(length: 8 + 1 + 8)
+    let sharedGroupFeedIndexInitializedKey = ValueBoxKey(length: 4 + 1)
+    let sharedChatListGroupHistoryInitializedKey = ValueBoxKey(length: 4 + 1)
+    let sharedPeerNextMessageIdByNamespaceKey = ValueBoxKey(length: 8 + 1 + 4)
+    let sharedBuffer = WriteBuffer()
+    
+    private var initializedChatList = Set<InitializedChatListKey>()
+    private var initializedHistoryPeerIds = Set<PeerId>()
+    private var initializedHistoryPeerIdTags: [PeerId: Set<MessageTags>] = [:]
+    private var initializedHistoryPeerIdCustomTags: [PeerId: Set<PeerIdThreadIdAndTag>] = [:]
+    private var reindexedHistoryPeerIdCustomTags: [PeerId: Set<PeerIdThreadIdAndTag>] = [:]
+    private var initializedGroupFeedIndexIds = Set<PeerGroupId>()
+    
+    private var peerNextMessageIdByNamespace: [PeerId: [MessageId.Namespace: MessageId.Id]] = [:]
+    private var updatedPeerNextMessageIdByNamespace: [PeerId: Set<MessageId.Namespace>] = [:]
+    
+    private var nextMessageStableId: UInt32?
+    private var nextMessageStableIdUpdated = false
+    
+    private var chatListTotalUnreadStates: [PeerGroupId: ChatListTotalUnreadState] = [:]
+    private var updatedChatListTotalUnreadStates = Set<PeerGroupId>()
+    
+    private var nextPeerOperationLogIndex: UInt32?
+    private var nextPeerOperationLogIndexUpdated = false
+    
+    private var currentPinnedChatPeerIds: Set<PeerId>?
+    private var currentPinnedChatPeerIdsUpdated = false
+    
+    private func peerHistoryInitializedKey(_ id: PeerId) -> ValueBoxKey {
+        self.sharedPeerHistoryInitializedKey.setInt64(0, value: id.toInt64())
+        self.sharedPeerHistoryInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryInitialized.rawValue)
+        return self.sharedPeerHistoryInitializedKey
+    }
+    
+    private func peerThreadHoleIndexInitializedKey(peerId: PeerId, threadId: Int64) -> ValueBoxKey {
+        self.sharedPeerThreadHoleIndexInitializedKey.setInt64(0, value: peerId.toInt64())
+        self.sharedPeerThreadHoleIndexInitializedKey.setInt8(8, value: MetadataPrefix.PeerHistoryThreadHoleIndexInitialized.rawValue)
+        self.sharedPeerThreadHoleIndexInitializedKey.setInt64(8 + 1, value: threadId)
+        return self.sharedPeerThreadHoleIndexInitializedKey
+    }
+    
+    private func peerHistoryInitializedTagKey(id: PeerId, tag: UInt32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 1 + 4)
+        key.setInt64(0, value: id.toInt64())
+        key.setInt8(8, value: MetadataPrefix.PeerHistoryTagInitialized.rawValue)
+        key.setUInt32(8 + 1, value: tag)
+        return key
+    }
+    
+    private func peerHistoryInitializedCustomTagKey(id: PeerId, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 1 + 8 + 4)
+        key.setInt64(0, value: id.toInt64())
+        key.setInt8(8, value: MetadataPrefix.PeerHistoryCustomTagInitialized.rawValue)
+        key.setInt64(8 + 1, value: threadId ?? 0)
+        key.setInt32(8 + 1 + 8, value: tag)
+        return key
+    }
+    
+    private func peerHistoryReindexedCustomTagKey(id: PeerId, threadId: Int64?, tag: Int32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 1 + 8 + 4)
+        key.setInt64(0, value: id.toInt64())
+        key.setInt8(8, value: MetadataPrefix.PeerHistoryCustomTagWithTagReindexed.rawValue)
+        key.setInt64(8 + 1, value: threadId ?? 0)
+        key.setInt32(8 + 1 + 8, value: tag)
+        return key
+    }
+    
+    private func peerHistoryInitializedCustomTagWithTagKey(id: PeerId, threadId: Int64?, tag: Int32, regularTag: UInt32) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 1 + 8 + 4 + 4)
+        key.setInt64(0, value: id.toInt64())
+        key.setInt8(8, value: MetadataPrefix.PeerHistoryCustomTagWithTagInitialized.rawValue)
+        key.setInt64(8 + 1, value: threadId ?? 0)
+        key.setInt32(8 + 1 + 8, value: tag)
+        key.setUInt32(8 + 1 + 8 + 4, value: regularTag)
+        return key
+    }
+    
+    private func groupFeedIndexInitializedKey(_ id: PeerGroupId) -> ValueBoxKey {
+        self.sharedGroupFeedIndexInitializedKey.setInt32(0, value: id.rawValue)
+        self.sharedGroupFeedIndexInitializedKey.setInt8(4, value: MetadataPrefix.GroupFeedIndexInitialized.rawValue)
+        return self.sharedGroupFeedIndexInitializedKey
+    }
+    
+    private func chatListGroupInitializedKey(_ key: InitializedChatListKey) -> ValueBoxKey {
+        self.sharedChatListGroupHistoryInitializedKey.setInt32(0, value: key.groupId.rawValue)
+        self.sharedChatListGroupHistoryInitializedKey.setInt8(4, value: MetadataPrefix.ChatListGroupInitialized.rawValue)
+        return self.sharedChatListGroupHistoryInitializedKey
+    }
+    
+    private func peerNextMessageIdByNamespaceKey(_ id: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        self.sharedPeerNextMessageIdByNamespaceKey.setInt64(0, value: id.toInt64())
+        self.sharedPeerNextMessageIdByNamespaceKey.setInt8(8, value: MetadataPrefix.PeerNextMessageIdByNamespace.rawValue)
+        self.sharedPeerNextMessageIdByNamespaceKey.setInt32(8 + 1, value: namespace)
+        
+        return self.sharedPeerNextMessageIdByNamespaceKey
+    }
+    
+    private func key(_ prefix: MetadataPrefix) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1)
+        key.setInt8(0, value: prefix.rawValue)
+        return key
+    }
+    
+    private func totalUnreadCountStateKey(groupId: PeerGroupId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1 + 4)
+        key.setInt8(0, value: MetadataPrefix.TotalUnreadCountStates.rawValue)
+        key.setInt32(1, value: groupId.rawValue)
+        return key
+    }
+    
+    private func totalUnreadCountLowerBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1)
+        key.setInt8(0, value: MetadataPrefix.TotalUnreadCountStates.rawValue)
+        return key
+    }
+    
+    private func totalUnreadCountUpperBound() -> ValueBoxKey {
+        return self.totalUnreadCountLowerBound().successor
+    }
+    
+    func setInitializedChatList(groupId: PeerGroupId) {
+        switch groupId {
+            case .root:
+                self.valueBox.set(self.table, key: self.key(MetadataPrefix.ChatListInitialized), value: MemoryBuffer())
+            case .group:
+                self.valueBox.set(self.table, key: self.chatListGroupInitializedKey(InitializedChatListKey(groupId: groupId)), value: MemoryBuffer())
+        }
+        self.initializedChatList.insert(InitializedChatListKey(groupId: groupId))
+    }
+    
+    func isInitializedChatList(groupId: PeerGroupId) -> Bool {
+        let key = InitializedChatListKey(groupId: groupId)
+        if self.initializedChatList.contains(key) {
+            return true
+        } else {
+            switch groupId {
+                case .root:
+                    if self.valueBox.exists(self.table, key: self.key(MetadataPrefix.ChatListInitialized)) {
+                        self.initializedChatList.insert(key)
+                        return true
+                    } else {
+                        return false
+                    }
+                case .group:
+                    if self.valueBox.exists(self.table, key: self.chatListGroupInitializedKey(key)) {
+                        self.initializedChatList.insert(key)
+                        return true
+                    } else {
+                        return false
+                    }
+            }
+        }
+    }
+    
+    func setShouldReindexUnreadCounts(value: Bool) {
+        if value {
+            self.valueBox.set(self.table, key: self.key(MetadataPrefix.ShouldReindexUnreadCounts), value: MemoryBuffer())
+        } else {
+            self.valueBox.remove(self.table, key: self.key(MetadataPrefix.ShouldReindexUnreadCounts), secure: false)
+        }
+    }
+    
+    func shouldReindexUnreadCounts() -> Bool {
+        if self.valueBox.exists(self.table, key: self.key(MetadataPrefix.ShouldReindexUnreadCounts)) {
+            return true
+        } else {
+            return false
+        }
+    }
+    
+    func setShouldReindexUnreadCountsState(value: Int32) {
+        var value = value
+        self.valueBox.set(self.table, key: self.key(MetadataPrefix.ShouldReindexUnreadCountsState), value: MemoryBuffer(memory: &value, capacity: 4, length: 4, freeWhenDone: false))
+       }
+    
+    func getShouldReindexUnreadCountsState() -> Int32? {
+        if let value = self.valueBox.get(self.table, key: self.key(MetadataPrefix.ShouldReindexUnreadCountsState)) {
+            var version: Int32 = 0
+            value.read(&version, offset: 0, length: 4)
+            return version
+        } else {
+            return nil
+        }
+    }
+    
+    func setInitialized(_ peerId: PeerId) {
+        self.initializedHistoryPeerIds.insert(peerId)
+        self.sharedBuffer.reset()
+        self.valueBox.set(self.table, key: self.peerHistoryInitializedKey(peerId), value: self.sharedBuffer)
+    }
+    
+    func isInitialized(_ peerId: PeerId) -> Bool {
+        if self.initializedHistoryPeerIds.contains(peerId) {
+            return true
+        } else {
+            if self.valueBox.exists(self.table, key: self.peerHistoryInitializedKey(peerId)) {
+                self.initializedHistoryPeerIds.insert(peerId)
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    func isThreadHoleIndexInitialized(peerId: PeerId, threadId: Int64) -> Bool {
+        if self.valueBox.exists(self.table, key: self.peerThreadHoleIndexInitializedKey(peerId: peerId, threadId: threadId)) {
+            return true
+        } else {
+            return false
+        }
+    }
+    
+    func setIsThreadHoleIndexInitialized(peerId: PeerId, threadId: Int64) {
+        self.valueBox.set(self.table, key: self.peerThreadHoleIndexInitializedKey(peerId: peerId, threadId: threadId), value: MemoryBuffer())
+    }
+    
+    func setPeerTagInitialized(peerId: PeerId, tag: MessageTags) {
+        if self.initializedHistoryPeerIdTags[peerId] == nil {
+            self.initializedHistoryPeerIdTags[peerId] = Set()
+        }
+        initializedHistoryPeerIdTags[peerId]!.insert(tag)
+        self.sharedBuffer.reset()
+        self.valueBox.set(self.table, key: self.peerHistoryInitializedTagKey(id: peerId, tag: tag.rawValue), value: self.sharedBuffer)
+    }
+    
+    func isPeerTagInitialized(peerId: PeerId, tag: MessageTags) -> Bool {
+        if let currentTags = self.initializedHistoryPeerIdTags[peerId], currentTags.contains(tag) {
+            return true
+        } else {
+            if self.valueBox.exists(self.table, key: self.peerHistoryInitializedTagKey(id: peerId, tag: tag.rawValue)) {
+                if self.initializedHistoryPeerIdTags[peerId] == nil {
+                    self.initializedHistoryPeerIdTags[peerId] = Set()
+                }
+                initializedHistoryPeerIdTags[peerId]!.insert(tag)
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    func setPeerCustomTagInitialized(peerId: PeerId, threadId: Int64?, tag: Int32) {
+        if self.initializedHistoryPeerIdCustomTags[peerId] == nil {
+            self.initializedHistoryPeerIdCustomTags[peerId] = Set()
+        }
+        self.initializedHistoryPeerIdCustomTags[peerId]!.insert(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: nil))
+        self.valueBox.set(self.table, key: self.peerHistoryInitializedCustomTagKey(id: peerId, threadId: threadId, tag: tag), value: MemoryBuffer())
+    }
+    
+    func isPeerCustomTagInitialized(peerId: PeerId, threadId: Int64?, tag: Int32) -> Bool {
+        if let currentTags = self.initializedHistoryPeerIdCustomTags[peerId], currentTags.contains(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: nil)) {
+            return true
+        } else {
+            if self.valueBox.exists(self.table, key: self.peerHistoryInitializedCustomTagKey(id: peerId, threadId: threadId, tag: tag)) {
+                if self.initializedHistoryPeerIdCustomTags[peerId] == nil {
+                    self.initializedHistoryPeerIdCustomTags[peerId] = Set()
+                }
+                self.initializedHistoryPeerIdCustomTags[peerId]!.insert(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: nil))
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    func isPeerCustomTagInitialized(peerId: PeerId, threadId: Int64?, tag: Int32, regularTag: UInt32) -> Bool {
+        if let currentTags = self.initializedHistoryPeerIdCustomTags[peerId], currentTags.contains(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: regularTag)) {
+            return true
+        } else {
+            if self.valueBox.exists(self.table, key: self.peerHistoryInitializedCustomTagWithTagKey(id: peerId, threadId: threadId, tag: tag, regularTag: regularTag)) {
+                if self.initializedHistoryPeerIdCustomTags[peerId] == nil {
+                    self.initializedHistoryPeerIdCustomTags[peerId] = Set()
+                }
+                self.initializedHistoryPeerIdCustomTags[peerId]!.insert(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: regularTag))
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    func setPeerCustomTagInitialized(peerId: PeerId, threadId: Int64?, tag: Int32, regularTag: UInt32) {
+        if self.initializedHistoryPeerIdCustomTags[peerId] == nil {
+            self.initializedHistoryPeerIdCustomTags[peerId] = Set()
+        }
+        self.initializedHistoryPeerIdCustomTags[peerId]!.insert(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: regularTag))
+        self.valueBox.set(self.table, key: self.peerHistoryInitializedCustomTagWithTagKey(id: peerId, threadId: threadId, tag: tag, regularTag: regularTag), value: MemoryBuffer())
+    }
+    
+    func setPeerCustomTagReindexed(peerId: PeerId, threadId: Int64?, tag: Int32) {
+        if self.reindexedHistoryPeerIdCustomTags[peerId] == nil {
+            self.reindexedHistoryPeerIdCustomTags[peerId] = Set()
+        }
+        self.reindexedHistoryPeerIdCustomTags[peerId]!.insert(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: nil))
+        self.valueBox.set(self.table, key: self.peerHistoryReindexedCustomTagKey(id: peerId, threadId: threadId, tag: tag), value: MemoryBuffer())
+    }
+    
+    func isPeerCustomTagReindexed(peerId: PeerId, threadId: Int64?, tag: Int32) -> Bool {
+        if let currentTags = self.reindexedHistoryPeerIdCustomTags[peerId], currentTags.contains(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: nil)) {
+            return true
+        } else {
+            if self.valueBox.exists(self.table, key: self.peerHistoryReindexedCustomTagKey(id: peerId, threadId: threadId, tag: tag)) {
+                if self.reindexedHistoryPeerIdCustomTags[peerId] == nil {
+                    self.reindexedHistoryPeerIdCustomTags[peerId] = Set()
+                }
+                self.reindexedHistoryPeerIdCustomTags[peerId]!.insert(PeerIdThreadIdAndTag(peerId: peerId, threadId: threadId, tag: tag, regularTag: nil))
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    func removePeerCustomTagInitializedList() {
+        self.initializedHistoryPeerIdCustomTags.removeAll()
+        
+        var removeKeys: [ValueBoxKey] = []
+        self.valueBox.scan(self.table, keys: { key in
+            if key.length == 8 + 1 + 8 + 4 {
+                if key.getInt8(8) == MetadataPrefix.PeerHistoryCustomTagInitialized.rawValue {
+                    removeKeys.append(key)
+                }
+            }
+            
+            return true
+        })
+        self.valueBox.scan(self.table, keys: { key in
+            if key.length == 8 + 1 + 8 + 4 {
+                if key.getInt8(8) == MetadataPrefix.PeerHistoryCustomTagWithTagInitialized.rawValue {
+                    removeKeys.append(key)
+                }
+            }
+            
+            return true
+        })
+        
+        for key in removeKeys {
+            self.valueBox.remove(self.table, key: key, secure: false)
+        }
+    }
+    
+    func setGroupFeedIndexInitialized(_ groupId: PeerGroupId) {
+        self.initializedGroupFeedIndexIds.insert(groupId)
+        self.sharedBuffer.reset()
+        self.valueBox.set(self.table, key: self.groupFeedIndexInitializedKey(groupId), value: self.sharedBuffer)
+    }
+    
+    func isGroupFeedIndexInitialized(_ groupId: PeerGroupId) -> Bool {
+        if self.initializedGroupFeedIndexIds.contains(groupId) {
+            return true
+        } else {
+            if self.valueBox.exists(self.table, key: self.groupFeedIndexInitializedKey(groupId)) {
+                self.initializedGroupFeedIndexIds.insert(groupId)
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+    
+    func getNextMessageIdAndIncrement(_ peerId: PeerId, namespace: MessageId.Namespace) -> MessageId {
+        if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId] {
+            if let nextId = messageIdByNamespace[namespace] {
+                self.peerNextMessageIdByNamespace[peerId]![namespace] = nextId + 1
+                if updatedPeerNextMessageIdByNamespace[peerId] != nil {
+                    updatedPeerNextMessageIdByNamespace[peerId]!.insert(namespace)
+                } else {
+                    updatedPeerNextMessageIdByNamespace[peerId] = Set<MessageId.Namespace>([namespace])
+                }
+                return MessageId(peerId: peerId, namespace: namespace, id: nextId)
+            } else {
+                var nextId: Int32 = 1
+                if let value = self.valueBox.get(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) {
+                    value.read(&nextId, offset: 0, length: 4)
+                }
+                self.peerNextMessageIdByNamespace[peerId]![namespace] = nextId + 1
+                if updatedPeerNextMessageIdByNamespace[peerId] != nil {
+                    updatedPeerNextMessageIdByNamespace[peerId]!.insert(namespace)
+                } else {
+                    updatedPeerNextMessageIdByNamespace[peerId] = Set<MessageId.Namespace>([namespace])
+                }
+                return MessageId(peerId: peerId, namespace: namespace, id: nextId)
+            }
+        } else {
+            var nextId: Int32 = 1
+            if let value = self.valueBox.get(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace)) {
+                value.read(&nextId, offset: 0, length: 4)
+            }
+            
+            self.peerNextMessageIdByNamespace[peerId] = [namespace: nextId + 1]
+            if updatedPeerNextMessageIdByNamespace[peerId] != nil {
+                updatedPeerNextMessageIdByNamespace[peerId]!.insert(namespace)
+            } else {
+                updatedPeerNextMessageIdByNamespace[peerId] = Set<MessageId.Namespace>([namespace])
+            }
+            return MessageId(peerId: peerId, namespace: namespace, id: nextId)
+        }
+    }
+    
+    func getNextStableMessageIndexId() -> UInt32 {
+        if let nextId = self.nextMessageStableId {
+            self.nextMessageStableId = nextId + 1
+            self.nextMessageStableIdUpdated = true
+            return nextId
+        } else {
+            if let value = self.valueBox.get(self.table, key: self.key(.NextStableMessageId)) {
+                var nextId: UInt32 = 0
+                value.read(&nextId, offset: 0, length: 4)
+                self.nextMessageStableId = nextId + 1
+                self.nextMessageStableIdUpdated = true
+                return nextId
+            } else {
+                let nextId: UInt32 = 1
+                self.nextMessageStableId = nextId + 1
+                self.nextMessageStableIdUpdated = true
+                return nextId
+            }
+        }
+    }
+    
+    func getNextPeerOperationLogIndex() -> UInt32 {
+        if let nextId = self.nextPeerOperationLogIndex {
+            self.nextPeerOperationLogIndex = nextId + 1
+            self.nextPeerOperationLogIndexUpdated = true
+            return nextId
+        } else {
+            if let value = self.valueBox.get(self.table, key: self.key(.NextPeerOperationLogIndex)) {
+                var nextId: UInt32 = 0
+                value.read(&nextId, offset: 0, length: 4)
+                self.nextPeerOperationLogIndex = nextId + 1
+                self.nextPeerOperationLogIndexUpdated = true
+                return nextId
+            } else {
+                let nextId: UInt32 = 1
+                self.nextPeerOperationLogIndex = nextId + 1
+                self.nextPeerOperationLogIndexUpdated = true
+                return nextId
+            }
+        }
+    }
+    
+    func removeAllTotalUnreadStates() {
+        var groupIds: [PeerGroupId] = []
+        self.valueBox.range(self.table, start: self.totalUnreadCountLowerBound(), end: self.totalUnreadCountUpperBound(), keys: { key in
+            let groupId = key.getInt32(1)
+            groupIds.append(PeerGroupId(rawValue: groupId))
+            return true
+        }, limit: 0)
+        for groupId in groupIds {
+            self.setTotalUnreadState(groupId: groupId, state: ChatListTotalUnreadState(absoluteCounters: [:], filteredCounters: [:]))
+        }
+    }
+    
+    func getTotalUnreadState(groupId: PeerGroupId) -> ChatListTotalUnreadState {
+        if let cached = self.chatListTotalUnreadStates[groupId] {
+            return cached
+        } else {
+            if let value = self.valueBox.get(self.table, key: self.totalUnreadCountStateKey(groupId: groupId)), let state = PostboxDecoder(buffer: value).decodeObjectForKey("_", decoder: { ChatListTotalUnreadState(decoder: $0) }) as? ChatListTotalUnreadState {
+                self.chatListTotalUnreadStates[groupId] = state
+                return state
+            } else {
+                let state = ChatListTotalUnreadState(absoluteCounters: [:], filteredCounters: [:])
+                self.chatListTotalUnreadStates[groupId] = state
+                return state
+            }
+        }
+    }
+    
+    func setTotalUnreadState(groupId: PeerGroupId, state: ChatListTotalUnreadState) {
+        let current = self.getTotalUnreadState(groupId: groupId)
+        if current != state {
+            self.chatListTotalUnreadStates[groupId] = state
+            self.updatedChatListTotalUnreadStates.insert(groupId)
+        }
+    }
+    
+    override func clearMemoryCache() {
+        self.initializedChatList.removeAll()
+        self.initializedHistoryPeerIds.removeAll()
+        self.peerNextMessageIdByNamespace.removeAll()
+        self.updatedPeerNextMessageIdByNamespace.removeAll()
+        self.nextMessageStableId = nil
+        self.nextMessageStableIdUpdated = false
+        self.chatListTotalUnreadStates.removeAll()
+        self.updatedChatListTotalUnreadStates.removeAll()
+    }
+    
+    override func beforeCommit() {
+        let sharedBuffer = WriteBuffer()
+        for (peerId, namespaces) in self.updatedPeerNextMessageIdByNamespace {
+            for namespace in namespaces {
+                if let messageIdByNamespace = self.peerNextMessageIdByNamespace[peerId], let maxId = messageIdByNamespace[namespace] {
+                    sharedBuffer.reset()
+                    var mutableMaxId = maxId
+                    sharedBuffer.write(&mutableMaxId, offset: 0, length: 4)
+                    self.valueBox.set(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace), value: sharedBuffer)
+                } else {
+                    self.valueBox.remove(self.table, key: self.peerNextMessageIdByNamespaceKey(peerId, namespace: namespace), secure: false)
+                }
+            }
+        }
+        self.updatedPeerNextMessageIdByNamespace.removeAll()
+        
+        if self.nextMessageStableIdUpdated {
+            if let nextMessageStableId = self.nextMessageStableId {
+                var nextId: UInt32 = nextMessageStableId
+                self.valueBox.set(self.table, key: self.key(.NextStableMessageId), value: MemoryBuffer(memory: &nextId, capacity: 4, length: 4, freeWhenDone: false))
+                self.nextMessageStableIdUpdated = false
+            }
+        }
+        
+        if self.nextPeerOperationLogIndexUpdated {
+            if let nextPeerOperationLogIndex = self.nextPeerOperationLogIndex {
+                var nextId: UInt32 = nextPeerOperationLogIndex
+                self.valueBox.set(self.table, key: self.key(.NextPeerOperationLogIndex), value: MemoryBuffer(memory: &nextId, capacity: 4, length: 4, freeWhenDone: false))
+                self.nextPeerOperationLogIndexUpdated = false
+            }
+        }
+        
+        for groupId in self.updatedChatListTotalUnreadStates {
+            if let state = self.chatListTotalUnreadStates[groupId] {
+                let buffer = PostboxEncoder()
+                buffer.encodeObject(state, forKey: "_")
+                self.valueBox.set(self.table, key: self.totalUnreadCountStateKey(groupId: groupId), value: buffer.readBufferNoCopy())
+            }
+            self.updatedChatListTotalUnreadStates.removeAll()
+        }
+    }
+    
+    func getNextCustomTagIdAndIncrement() -> Int32 {
+        var nextId: Int32 = 1
+        if let value = self.valueBox.get(self.table, key: self.key(.NextCustomTagId)) {
+            value.read(&nextId, offset: 0, length: 4)
+        }
+        var storeNextId = nextId + 1
+        self.valueBox.set(self.table, key: self.key(.NextCustomTagId), value: MemoryBuffer(memory: &storeNextId, capacity: 4, length: 4, freeWhenDone: false))
+        
+        return nextId
+    }
+}

submodules/Postbox/Sources/MessageHistoryOperation.swift

@@ -0,0 +1,10 @@
+import Foundation
+
+enum MessageHistoryOperation {
+    case InsertMessage(IntermediateMessage)
+    case Remove([(MessageIndex, MessageTags)])
+    case UpdateReadState(PeerId, CombinedPeerReadState)
+    case UpdateEmbeddedMedia(MessageIndex, ReadBuffer)
+    case UpdateTimestamp(MessageIndex, Int32)
+    case UpdateGroupInfos([MessageId: MessageGroupInfo])
+}

submodules/Postbox/Sources/MessageHistoryReadStateTable.swift

@@ -0,0 +1,576 @@
+import Foundation
+
+private let traceReadStates = false
+
+enum ApplyInteractiveMaxReadIdResult {
+    case None
+    case Push(thenSync: Bool)
+}
+
+private final class InternalPeerReadStates {
+    var namespaces: [MessageId.Namespace: PeerReadState]
+    
+    init(namespaces: [MessageId.Namespace: PeerReadState]) {
+        self.namespaces = namespaces
+    }
+}
+
+final class MessageHistoryReadStateTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: false)
+    }
+    
+    private let seedConfiguration: SeedConfiguration
+    
+    private var cachedPeerReadStates: [PeerId: InternalPeerReadStates?] = [:]
+    private var updatedInitialPeerReadStates: [PeerId: [MessageId.Namespace: PeerReadState]] = [:]
+    
+    private let sharedKey = ValueBoxKey(length: 8)
+    
+    private func key(_ id: PeerId) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: id.toInt64())
+        return self.sharedKey
+    }
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, seedConfiguration: SeedConfiguration) {
+        self.seedConfiguration = seedConfiguration
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+
+    private func get(_ id: PeerId) -> InternalPeerReadStates? {
+        if let states = self.cachedPeerReadStates[id] {
+            return states
+        } else {
+            if let value = self.valueBox.get(self.table, key: self.key(id)) {
+                var count: Int32 = 0
+                value.read(&count, offset: 0, length: 4)
+                var stateByNamespace: [MessageId.Namespace: PeerReadState] = [:]
+                for _ in 0 ..< count {
+                    var namespaceId: Int32 = 0
+                    value.read(&namespaceId, offset: 0, length: 4)
+                    
+                    let state: PeerReadState
+                    var kind: Int8 = 0
+                    value.read(&kind, offset: 0, length: 1)
+                    if kind == 0 {
+                        var maxIncomingReadId: Int32 = 0
+                        var maxOutgoingReadId: Int32 = 0
+                        var maxKnownId: Int32 = 0
+                        var count: Int32 = 0
+                        
+                        value.read(&maxIncomingReadId, offset: 0, length: 4)
+                        value.read(&maxOutgoingReadId, offset: 0, length: 4)
+                        value.read(&maxKnownId, offset: 0, length: 4)
+                        value.read(&count, offset: 0, length: 4)
+                        
+                        var flags: Int32 = 0
+                        value.read(&flags, offset: 0, length: 4)
+                        let markedUnread = (flags & (1 << 0)) != 0
+                        
+                        state = .idBased(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count, markedUnread: markedUnread)
+                    } else {
+                        var maxIncomingReadTimestamp: Int32 = 0
+                        var maxIncomingReadIdPeerId: Int64 = 0
+                        var maxIncomingReadIdNamespace: Int32 = 0
+                        var maxIncomingReadIdId: Int32 = 0
+                        
+                        var maxOutgoingReadTimestamp: Int32 = 0
+                        var maxOutgoingReadIdPeerId: Int64 = 0
+                        var maxOutgoingReadIdNamespace: Int32 = 0
+                        var maxOutgoingReadIdId: Int32 = 0
+                        
+                        var count: Int32 = 0
+                        
+                        value.read(&maxIncomingReadTimestamp, offset: 0, length: 4)
+                        value.read(&maxIncomingReadIdPeerId, offset: 0, length: 8)
+                        value.read(&maxIncomingReadIdNamespace, offset: 0, length: 4)
+                        value.read(&maxIncomingReadIdId, offset: 0, length: 4)
+                        
+                        value.read(&maxOutgoingReadTimestamp, offset: 0, length: 4)
+                        value.read(&maxOutgoingReadIdPeerId, offset: 0, length: 8)
+                        value.read(&maxOutgoingReadIdNamespace, offset: 0, length: 4)
+                        value.read(&maxOutgoingReadIdId, offset: 0, length: 4)
+                        
+                        value.read(&count, offset: 0, length: 4)
+                        
+                        var flags: Int32 = 0
+                        value.read(&flags, offset: 0, length: 4)
+                        let markedUnread = (flags & (1 << 0)) != 0
+                        
+                        state = .indexBased(maxIncomingReadIndex: MessageIndex(id: MessageId(peerId: PeerId(maxIncomingReadIdPeerId), namespace: maxIncomingReadIdNamespace, id: maxIncomingReadIdId), timestamp: maxIncomingReadTimestamp), maxOutgoingReadIndex: MessageIndex(id: MessageId(peerId: PeerId(maxOutgoingReadIdPeerId), namespace: maxOutgoingReadIdNamespace, id: maxOutgoingReadIdId), timestamp: maxOutgoingReadTimestamp), count: count, markedUnread: markedUnread)
+                    }
+                    stateByNamespace[namespaceId] = state
+                }
+                let states = InternalPeerReadStates(namespaces: stateByNamespace)
+                self.cachedPeerReadStates[id] = states
+                return states
+            } else {
+                self.cachedPeerReadStates[id] = nil
+                return nil
+            }
+        }
+    }
+    
+    func getCombinedState(_ peerId: PeerId) -> CombinedPeerReadState? {
+        if let states = self.get(peerId) {
+            return CombinedPeerReadState(states: states.namespaces.map({$0}))
+        }
+        return nil
+    }
+    
+    private func markReadStatesAsUpdated(_ peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) {
+        if self.updatedInitialPeerReadStates[peerId] == nil {
+            self.updatedInitialPeerReadStates[peerId] = namespaces
+        }
+    }
+    
+    func resetStates(_ peerId: PeerId, namespaces: [MessageId.Namespace: PeerReadState]) -> CombinedPeerReadState? {
+        if traceReadStates {
+            print("[ReadStateTable] resetStates peerId: \(peerId), namespaces: \(namespaces)")
+        }
+        
+        if let states = self.get(peerId) {
+            var updated = false
+            for (namespace, state) in namespaces {
+                if states.namespaces[namespace] == nil || states.namespaces[namespace]! != state {
+                    self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces)
+                    updated = true
+                }
+                states.namespaces[namespace] = state
+            }
+            if updated {
+                return CombinedPeerReadState(states: states.namespaces.map({$0}))
+            } else {
+                return nil
+            }
+        } else {
+            self.markReadStatesAsUpdated(peerId, namespaces: [:])
+            let states = InternalPeerReadStates(namespaces: namespaces)
+            self.cachedPeerReadStates[peerId] = states
+            return CombinedPeerReadState(states: states.namespaces.map({$0}))
+        }
+    }
+    
+    
+    func addIncomingMessages(_ peerId: PeerId, indices: Set<MessageIndex>) -> (CombinedPeerReadState?, Bool) {
+        var indicesByNamespace: [MessageId.Namespace: [MessageIndex]] = [:]
+        for index in indices {
+            if indicesByNamespace[index.id.namespace] != nil {
+                indicesByNamespace[index.id.namespace]!.append(index)
+            } else {
+                indicesByNamespace[index.id.namespace] = [index]
+            }
+        }
+        
+        if let states = self.get(peerId) {
+            if traceReadStates {
+                print("[ReadStateTable] addIncomingMessages peerId: \(peerId), indices: \(indices) (before: \(states.namespaces))")
+            }
+            
+            var updated = false
+            let invalidated = false
+            for (namespace, namespaceIndices) in indicesByNamespace {
+                let currentState = states.namespaces[namespace] ?? self.seedConfiguration.defaultMessageNamespaceReadStates[namespace]
+                
+                if let currentState = currentState {
+                    var addedUnreadCount: Int32 = 0
+                    for index in namespaceIndices {
+                        switch currentState {
+                            case let .idBased(maxIncomingReadId, _, maxKnownId, _, _):
+                                if index.id.id > maxKnownId && index.id.id > maxIncomingReadId {
+                                    addedUnreadCount += 1
+                                }
+                            case let .indexBased(maxIncomingReadIndex, _, _, _):
+                                if index > maxIncomingReadIndex {
+                                    addedUnreadCount += 1
+                                }
+                        }
+                    }
+                    
+                    if addedUnreadCount != 0 {
+                        self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces)
+                        
+                        states.namespaces[namespace] = currentState.withAddedCount(addedUnreadCount)
+                        updated = true
+                        
+                        if traceReadStates {
+                            print("[ReadStateTable] added \(addedUnreadCount)")
+                        }
+                    }
+                }
+            }
+            
+            return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidated)
+        } else {
+            if traceReadStates {
+                print("[ReadStateTable] addIncomingMessages peerId: \(peerId), just invalidated)")
+            }
+            return (nil, true)
+        }
+    }
+    
+    func deleteMessages(_ peerId: PeerId, indices: [MessageIndex], incomingStatsInIndices: (PeerId, MessageId.Namespace, [MessageIndex]) -> (Int, Bool)) -> (CombinedPeerReadState?, Bool) {
+        var indicesByNamespace: [MessageId.Namespace: [MessageIndex]] = [:]
+        for index in indices {
+            if indicesByNamespace[index.id.namespace] != nil {
+                indicesByNamespace[index.id.namespace]!.append(index)
+            } else {
+                indicesByNamespace[index.id.namespace] = [index]
+            }
+        }
+        
+        if let states = self.get(peerId) {
+            if traceReadStates {
+                print("[ReadStateTable] deleteMessages peerId: \(peerId), ids: \(indices) (before: \(states.namespaces))")
+            }
+            
+            var updated = false
+            var invalidate = false
+            for (namespace, namespaceIndices) in indicesByNamespace {
+                if let currentState = states.namespaces[namespace] {
+                    var unreadIndices: [MessageIndex] = []
+                    for index in namespaceIndices {
+                        if !currentState.isIncomingMessageIndexRead(index) {
+                            unreadIndices.append(index)
+                        }
+                    }
+                    
+                    let (knownCount, holes) = incomingStatsInIndices(peerId, namespace, unreadIndices)
+                    if holes {
+                        invalidate = true
+                    }
+                    
+                    self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces)
+                    
+                    var updatedState = currentState.withAddedCount(Int32(-knownCount))
+                    if updatedState.count < 0 {
+                        invalidate = true
+                        updatedState = currentState.withAddedCount(-updatedState.count)
+                    }
+                    
+                    states.namespaces[namespace] = updatedState
+                    updated = true
+                } else {
+                    invalidate = true
+                }
+            }
+            
+            return (updated ? CombinedPeerReadState(states: states.namespaces.map({$0})) : nil, invalidate)
+        } else {
+            return (nil, true)
+        }
+    }
+    
+    func applyIncomingMaxReadId(_ messageId: MessageId, incomingStatsInRange: (MessageId.Namespace, MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), topMessageId: (MessageId.Id, Bool)?) -> (CombinedPeerReadState?, Bool) {
+        if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] {
+            if traceReadStates {
+                print("[ReadStateTable] applyMaxReadId peerId: \(messageId.peerId), maxReadId: \(messageId) (before: \(states.namespaces))")
+            }
+            
+            switch state {
+                case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count, markedUnread):
+                    if maxIncomingReadId < messageId.id || (topMessageId != nil && (messageId.id == topMessageId!.0 || topMessageId!.1) && state.count != 0) || markedUnread {
+                        var (deltaCount, holes) = incomingStatsInRange(messageId.namespace, maxIncomingReadId + 1, messageId.id)
+                        
+                        if traceReadStates {
+                            print("[ReadStateTable] applyMaxReadId after deltaCount: \(deltaCount), holes: \(holes)")
+                        }
+                        
+                        if let topMessageId = topMessageId, (messageId.id == topMessageId.0 || topMessageId.1) {
+                            if deltaCount != Int(state.count) {
+                                deltaCount = Int(state.count)
+                                holes = true
+                            }
+                        }
+                        
+                        self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces)
+                        
+                        states.namespaces[messageId.namespace] = .idBased(maxIncomingReadId: messageId.id, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: max(0, count - Int32(deltaCount)), markedUnread: false)
+                        return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes)
+                }
+                case .indexBased:
+                    assertionFailure()
+                    break
+            }
+        } else {
+            return (nil, true)
+        }
+        
+        return (nil, false)
+    }
+    
+    func applyIncomingMaxReadIndex(_ messageIndex: MessageIndex, topMessageIndex: MessageIndex?, incomingStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMessageIds: [MessageId])) -> (CombinedPeerReadState?, Bool, [MessageId]) {
+        if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] {
+            if traceReadStates {
+                print("[ReadStateTable] applyIncomingMaxReadIndex peerId: \(messageIndex.id.peerId), maxReadIndex: \(messageIndex) (before: \(states.namespaces))")
+            }
+            
+            switch state {
+                case .idBased:
+                    assertionFailure()
+                case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count, markedUnread):
+                    var readPastTopIndex = false
+                    if let topMessageIndex = topMessageIndex, messageIndex >= topMessageIndex && count != 0 {
+                        readPastTopIndex = true
+                    }
+                    if maxIncomingReadIndex < messageIndex || markedUnread || readPastTopIndex {
+                        let (realDeltaCount, holes, messageIds) = incomingStatsInRange(maxIncomingReadIndex.peerLocalSuccessor(), messageIndex)
+                        var deltaCount = realDeltaCount
+                        if readPastTopIndex {
+                            deltaCount = max(Int(count), deltaCount)
+                        }
+                        
+                        if traceReadStates {
+                            print("[ReadStateTable] applyIncomingMaxReadIndex after deltaCount: \(deltaCount), holes: \(holes)")
+                        }
+                        
+                        self.markReadStatesAsUpdated(messageIndex.id.peerId, namespaces: states.namespaces)
+                        
+                        states.namespaces[messageIndex.id.namespace] = .indexBased(maxIncomingReadIndex: messageIndex, maxOutgoingReadIndex: maxOutgoingReadIndex, count: max(0, count - Int32(deltaCount)), markedUnread: false)
+                        return (CombinedPeerReadState(states: states.namespaces.map({$0})), holes, messageIds)
+                    }
+            }
+        } else {
+            return (nil, true, [])
+        }
+        
+        return (nil, false, [])
+    }
+    
+    func applyOutgoingMaxReadId(_ messageId: MessageId) -> (CombinedPeerReadState?, Bool) {
+        if let states = self.get(messageId.peerId), let state = states.namespaces[messageId.namespace] {
+            switch state {
+                case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count, markedUnread):
+                    if maxOutgoingReadId < messageId.id {
+                        self.markReadStatesAsUpdated(messageId.peerId, namespaces: states.namespaces)
+                        states.namespaces[messageId.namespace] = .idBased(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: messageId.id, maxKnownId: maxKnownId, count: count, markedUnread: markedUnread)
+                        return (CombinedPeerReadState(states: states.namespaces.map({$0})), false)
+                    }
+                case .indexBased:
+                    assertionFailure()
+                    break
+            }
+        } else {
+            return (nil, true)
+        }
+        
+        return (nil, false)
+    }
+    
+    func applyOutgoingMaxReadIndex(_ messageIndex: MessageIndex, outgoingIndexStatsInRange: (MessageIndex, MessageIndex) -> [MessageId]) -> (CombinedPeerReadState?, Bool, [MessageId]) {
+        if let states = self.get(messageIndex.id.peerId), let state = states.namespaces[messageIndex.id.namespace] {
+            switch state {
+                case .idBased:
+                    assertionFailure()
+                    break
+                case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count, markedUnread):
+                    if maxOutgoingReadIndex < messageIndex {
+                        let messageIds: [MessageId] = outgoingIndexStatsInRange(maxOutgoingReadIndex.peerLocalSuccessor(), messageIndex)
+                        
+                        self.markReadStatesAsUpdated(messageIndex.id.peerId, namespaces: states.namespaces)
+                        states.namespaces[messageIndex.id.namespace] = .indexBased(maxIncomingReadIndex: maxIncomingReadIndex, maxOutgoingReadIndex: messageIndex, count: count, markedUnread: markedUnread)
+                        return (CombinedPeerReadState(states: states.namespaces.map({$0})), false, messageIds)
+                    }
+            }
+        } else {
+            return (nil, true, [])
+        }
+        
+        return (nil, false, [])
+    }
+    
+    func applyInteractiveMaxReadIndex(postbox: PostboxImpl, messageIndex: MessageIndex, incomingStatsInRange: (MessageId.Namespace, MessageId.Id, MessageId.Id) -> (count: Int, holes: Bool), incomingIndexStatsInRange: (MessageIndex, MessageIndex) -> (count: Int, holes: Bool, readMessageIds: [MessageId]), topMessageId: (MessageId.Id, Bool)?, topMessageIndexByNamespace: (MessageId.Namespace) -> MessageIndex?) -> (combinedState: CombinedPeerReadState?, ApplyInteractiveMaxReadIdResult, readMessageIds: [MessageId]) {
+        if let states = self.get(messageIndex.id.peerId) {
+            if let state = states.namespaces[messageIndex.id.namespace] {
+                switch state {
+                    case .idBased:
+                        let (combinedState, holes) = self.applyIncomingMaxReadId(messageIndex.id, incomingStatsInRange: incomingStatsInRange, topMessageId: topMessageId)
+                        
+                        if let combinedState = combinedState {
+                            return (combinedState, .Push(thenSync: holes), [])
+                        }
+                        
+                        return (combinedState, holes ? .Push(thenSync: true) : .None, [])
+                    case .indexBased:
+                        let topMessageIndex: MessageIndex? = topMessageIndexByNamespace(messageIndex.id.namespace)
+                        let (combinedState, holes, messageIds) = self.applyIncomingMaxReadIndex(messageIndex, topMessageIndex: topMessageIndex, incomingStatsInRange: incomingIndexStatsInRange)
+                        
+                        if let combinedState = combinedState {
+                            return (combinedState, .Push(thenSync: holes), messageIds)
+                        }
+                        
+                        return (combinedState, holes ? .Push(thenSync: true) : .None, messageIds)
+                }
+            } else {
+                for (namespace, state) in states.namespaces {
+                    if let topIndex = topMessageIndexByNamespace(namespace), topIndex <= messageIndex {
+                        switch state {
+                            case .idBased:
+                                let (combinedState, holes) = self.applyIncomingMaxReadId(topIndex.id, incomingStatsInRange: incomingStatsInRange, topMessageId: nil)
+                                
+                                if let combinedState = combinedState {
+                                    return (combinedState, .Push(thenSync: holes), [])
+                                }
+                                
+                                return (combinedState, holes ? .Push(thenSync: true) : .None, [])
+                            case .indexBased:
+                                let (combinedState, holes, messageIds) = self.applyIncomingMaxReadIndex(topIndex, topMessageIndex: topMessageIndexByNamespace(namespace), incomingStatsInRange: incomingIndexStatsInRange)
+                                
+                                if let combinedState = combinedState {
+                                    return (combinedState, .Push(thenSync: holes), messageIds)
+                                }
+                                
+                                return (combinedState, holes ? .Push(thenSync: true) : .None, messageIds)
+                        }
+                    }
+                }
+                return (nil, .Push(thenSync: true), [])
+            }
+        } else {
+            return (nil, .Push(thenSync: true), [])
+        }
+    }
+    
+    func applyInteractiveMarkUnread(peerId: PeerId, namespace: MessageId.Namespace, value: Bool) -> CombinedPeerReadState? {
+        if let states = self.get(peerId), let state = states.namespaces[namespace] {
+            switch state {
+                case let .idBased(maxIncomingReadId, maxOutgoingReadId, maxKnownId, count, markedUnread):
+                    if markedUnread != value {
+                        self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces)
+                        
+                        states.namespaces[namespace] = .idBased(maxIncomingReadId: maxIncomingReadId, maxOutgoingReadId: maxOutgoingReadId, maxKnownId: maxKnownId, count: count, markedUnread: value)
+                        return CombinedPeerReadState(states: states.namespaces.map({$0}))
+                    } else {
+                        return nil
+                    }
+                case let .indexBased(maxIncomingReadIndex, maxOutgoingReadIndex, count, markedUnread):
+                    if markedUnread != value {
+                        self.markReadStatesAsUpdated(peerId, namespaces: states.namespaces)
+                        
+                        states.namespaces[namespace] = .indexBased(maxIncomingReadIndex: maxIncomingReadIndex, maxOutgoingReadIndex: maxOutgoingReadIndex, count: count, markedUnread: value)
+                        return CombinedPeerReadState(states: states.namespaces.map({$0}))
+                    } else {
+                        return nil
+                    }
+            }
+        } else {
+            return nil
+        }
+    }
+    
+    func transactionUnreadCountDeltas() -> [PeerId: Int32] {
+        var deltas: [PeerId: Int32] = [:]
+        for (id, initialNamespaces) in self.updatedInitialPeerReadStates {
+            var initialCount: Int32 = 0
+            for (_, state) in initialNamespaces {
+                initialCount += state.count
+            }
+            
+            var updatedCount: Int32 = 0
+            if let maybeStates = self.cachedPeerReadStates[id] {
+                if let states = maybeStates {
+                    for (_, state) in states.namespaces {
+                        updatedCount += state.count
+                    }
+                }
+            } else {
+                assertionFailure()
+            }
+            
+            if initialCount != updatedCount {
+                deltas[id] = updatedCount - initialCount
+            }
+        }
+        return deltas
+    }
+    
+    func transactionAlteredInitialPeerCombinedReadStates() -> [PeerId: CombinedPeerReadState] {
+        var result: [PeerId: CombinedPeerReadState] = [:]
+        for (peerId, namespacesAndStates) in self.updatedInitialPeerReadStates {
+            var states: [(MessageId.Namespace, PeerReadState)] = []
+            for (namespace, state) in namespacesAndStates {
+                states.append((namespace, state))
+            }
+            result[peerId] = CombinedPeerReadState(states: states)
+        }
+        return result
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedPeerReadStates.removeAll()
+        assert(self.updatedInitialPeerReadStates.isEmpty)
+    }
+    
+    override func beforeCommit() {
+        if !self.updatedInitialPeerReadStates.isEmpty {
+            let sharedBuffer = WriteBuffer()
+            for (id, _) in self.updatedInitialPeerReadStates {
+                if let wrappedStates = self.cachedPeerReadStates[id], let states = wrappedStates {
+                    sharedBuffer.reset()
+                    var count: Int32 = Int32(states.namespaces.count)
+                    sharedBuffer.write(&count, offset: 0, length: 4)
+                    for (namespace, state) in states.namespaces {
+                        var namespaceId: Int32 = namespace
+                        sharedBuffer.write(&namespaceId, offset: 0, length: 4)
+                        
+                        switch state {
+                            case .idBased(var maxIncomingReadId, var maxOutgoingReadId, var maxKnownId, var count, let markedUnread):
+                                var kind: Int8 = 0
+                                sharedBuffer.write(&kind, offset: 0, length: 1)
+                                
+                                sharedBuffer.write(&maxIncomingReadId, offset: 0, length: 4)
+                                sharedBuffer.write(&maxOutgoingReadId, offset: 0, length: 4)
+                                sharedBuffer.write(&maxKnownId, offset: 0, length: 4)
+                                sharedBuffer.write(&count, offset: 0, length: 4)
+                                var flags: Int32 = 0
+                                if markedUnread {
+                                    flags |= (1 << 0)
+                                }
+                                sharedBuffer.write(&flags, offset: 0, length: 4)
+                            case .indexBased(let maxIncomingReadIndex, let maxOutgoingReadIndex, var count, let markedUnread):
+                                var kind: Int8 = 1
+                                sharedBuffer.write(&kind, offset: 0, length: 1)
+                            
+                                var maxIncomingReadTimestamp: Int32 = maxIncomingReadIndex.timestamp
+                                var maxIncomingReadIdPeerId: Int64 = maxIncomingReadIndex.id.peerId.toInt64()
+                                var maxIncomingReadIdNamespace: Int32 = maxIncomingReadIndex.id.namespace
+                                var maxIncomingReadIdId: Int32 = maxIncomingReadIndex.id.id
+                                
+                                var maxOutgoingReadTimestamp: Int32 = maxOutgoingReadIndex.timestamp
+                                var maxOutgoingReadIdPeerId: Int64 = maxOutgoingReadIndex.id.peerId.toInt64()
+                                var maxOutgoingReadIdNamespace: Int32 = maxOutgoingReadIndex.id.namespace
+                                var maxOutgoingReadIdId: Int32 = maxOutgoingReadIndex.id.id
+                                
+                                sharedBuffer.write(&maxIncomingReadTimestamp, offset: 0, length: 4)
+                                sharedBuffer.write(&maxIncomingReadIdPeerId, offset: 0, length: 8)
+                                sharedBuffer.write(&maxIncomingReadIdNamespace, offset: 0, length: 4)
+                                sharedBuffer.write(&maxIncomingReadIdId, offset: 0, length: 4)
+                                
+                                sharedBuffer.write(&maxOutgoingReadTimestamp, offset: 0, length: 4)
+                                sharedBuffer.write(&maxOutgoingReadIdPeerId, offset: 0, length: 8)
+                                sharedBuffer.write(&maxOutgoingReadIdNamespace, offset: 0, length: 4)
+                                sharedBuffer.write(&maxOutgoingReadIdId, offset: 0, length: 4)
+                            
+                                sharedBuffer.write(&count, offset: 0, length: 4)
+                            
+                                var flags: Int32 = 0
+                                if markedUnread {
+                                    flags |= 1 << 0
+                                }
+                                sharedBuffer.write(&flags, offset: 0, length: 4)
+                        }
+                    }
+                    self.valueBox.set(self.table, key: self.key(id), value: sharedBuffer)
+                } else {
+                    self.valueBox.remove(self.table, key: self.key(id), secure: false)
+                }
+            }
+            self.updatedInitialPeerReadStates.removeAll()
+
+            if !self.useCaches {
+                self.cachedPeerReadStates.removeAll()
+            }
+        }
+    }
+}

submodules/Postbox/Sources/MessageHistorySavedMessagesIndexView.swift

@@ -0,0 +1,224 @@
+import Foundation
+
+final class MutableMessageHistorySavedMessagesIndexView: MutablePostboxView {
+    final class Item {
+        let id: Int64
+        let peer: Peer?
+        let pinnedIndex: Int?
+        let index: MessageIndex
+        let topMessage: Message?
+        let unreadCount: Int
+        let markedUnread: Bool
+        let embeddedInterfaceState: StoredPeerChatInterfaceState?
+        
+        init(
+            id: Int64,
+            peer: Peer?,
+            pinnedIndex: Int?,
+            index: MessageIndex,
+            topMessage: Message?,
+            unreadCount: Int,
+            markedUnread: Bool,
+            embeddedInterfaceState: StoredPeerChatInterfaceState?
+        ) {
+            self.id = id
+            self.peer = peer
+            self.pinnedIndex = pinnedIndex
+            self.index = index
+            self.topMessage = topMessage
+            self.unreadCount = unreadCount
+            self.markedUnread = markedUnread
+            self.embeddedInterfaceState = embeddedInterfaceState
+        }
+    }
+    
+    fileprivate let peerId: PeerId
+    fileprivate var peer: Peer?
+    fileprivate var items: [Item] = []
+    private var hole: ForumTopicListHolesEntry?
+    fileprivate var isLoading: Bool = false
+    
+    init(postbox: PostboxImpl, peerId: PeerId) {
+        self.peerId = peerId
+        
+        self.reload(postbox: postbox)
+    }
+    
+    private func reload(postbox: PostboxImpl) {
+        self.items.removeAll()
+        
+        self.peer = postbox.peerTable.get(self.peerId)
+        
+        let validIndexBoundary = postbox.peerThreadCombinedStateTable.get(peerId: self.peerId)?.validIndexBoundary
+        self.isLoading = validIndexBoundary == nil
+        
+        if let validIndexBoundary = validIndexBoundary {
+            if validIndexBoundary.messageId != 1 {
+                self.hole = ForumTopicListHolesEntry(peerId: self.peerId, index: validIndexBoundary)
+            } else {
+                self.hole = nil
+            }
+        } else {
+            self.hole = ForumTopicListHolesEntry(peerId: self.peerId, index: nil)
+        }
+        
+        if !self.isLoading {
+            let pinnedThreadIds = postbox.messageHistoryThreadPinnedTable.get(peerId: self.peerId)
+        
+            for item in postbox.messageHistoryThreadIndexTable.getAll(peerId: self.peerId) {
+                var pinnedIndex: Int?
+                if let index = pinnedThreadIds.firstIndex(of: item.threadId) {
+                    pinnedIndex = index
+                }
+                
+                let embeddedInterfaceState = postbox.peerChatThreadInterfaceStateTable.get(PeerChatThreadId(peerId: self.peerId, threadId: item.threadId))
+                
+                self.items.append(Item(
+                    id: item.threadId,
+                    peer: postbox.peerTable.get(PeerId(item.threadId)),
+                    pinnedIndex: pinnedIndex,
+                    index: item.index,
+                    topMessage: postbox.getMessage(item.index.id),
+                    unreadCount: Int(item.info.summary.totalUnreadCount),
+                    markedUnread: item.info.summary.isMarkedUnread,
+                    embeddedInterfaceState: embeddedInterfaceState
+                ))
+            }
+            
+            self.items.sort(by: { lhs, rhs in
+                if let lhsPinnedIndex = lhs.pinnedIndex, let rhsPinnedIndex = rhs.pinnedIndex {
+                    return lhsPinnedIndex < rhsPinnedIndex
+                } else if (lhs.pinnedIndex == nil) != (rhs.pinnedIndex == nil) {
+                    if lhs.pinnedIndex != nil {
+                        return true
+                    } else {
+                        return false
+                    }
+                }
+                
+                return lhs.index > rhs.index
+            })
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if transaction.updatedMessageThreadPeerIds.contains(self.peerId) || transaction.updatedPinnedThreads.contains(self.peerId) || transaction.updatedPeerThreadCombinedStates.contains(self.peerId) || transaction.currentUpdatedMessageTagSummaries.contains(where: { $0.key.peerId == self.peerId }) || transaction.currentUpdatedMessageActionsSummaries.contains(where: { $0.key.peerId == self.peerId }) || transaction.currentUpdatedPeerChatListEmbeddedStates.contains(self.peerId) || transaction.currentUpdatedPeerNotificationSettings[self.peerId] != nil || transaction.updatedPinnedThreads.contains(self.peerId) {
+            self.reload(postbox: postbox)
+            updated = true
+        }
+        
+        return updated
+    }
+    
+    func topHole() -> ForumTopicListHolesEntry? {
+        return self.hole
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        self.reload(postbox: postbox)
+        
+        return true
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageHistorySavedMessagesIndexView(self)
+    }
+}
+
+public final class EngineMessageHistorySavedMessagesThread {
+    public final class Item: Equatable {
+        public let id: Int64
+        public let peer: Peer?
+        public let pinnedIndex: Int?
+        public let index: MessageIndex
+        public let topMessage: Message?
+        public let unreadCount: Int
+        public let markedUnread: Bool
+        public let embeddedInterfaceState: StoredPeerChatInterfaceState?
+        
+        public init(
+            id: Int64,
+            peer: Peer?,
+            pinnedIndex: Int?,
+            index: MessageIndex,
+            topMessage: Message?,
+            unreadCount: Int,
+            markedUnread: Bool,
+            embeddedInterfaceState: StoredPeerChatInterfaceState?
+        ) {
+            self.id = id
+            self.peer = peer
+            self.pinnedIndex = pinnedIndex
+            self.index = index
+            self.topMessage = topMessage
+            self.unreadCount = unreadCount
+            self.markedUnread = markedUnread
+            self.embeddedInterfaceState = embeddedInterfaceState
+        }
+        
+        public static func ==(lhs: Item, rhs: Item) -> Bool {
+            if lhs.id != rhs.id {
+                return false
+            }
+            if !arePeersEqual(lhs.peer, rhs.peer) {
+                return false
+            }
+            if lhs.pinnedIndex != rhs.pinnedIndex {
+                return false
+            }
+            if lhs.index != rhs.index {
+                return false
+            }
+            if let lhsMessage = lhs.topMessage, let rhsMessage = rhs.topMessage {
+                if lhsMessage.index != rhsMessage.index {
+                    return false
+                }
+                if lhsMessage.stableVersion != rhsMessage.stableVersion {
+                    return false
+                }
+            } else if (lhs.topMessage == nil) != (rhs.topMessage == nil) {
+                return false
+            }
+            if lhs.unreadCount != rhs.unreadCount {
+                return false
+            }
+            if lhs.markedUnread != rhs.markedUnread {
+                return false
+            }
+            if lhs.embeddedInterfaceState != rhs.embeddedInterfaceState {
+                return false
+            }
+            
+            return true
+        }
+    }
+}
+
+public final class MessageHistorySavedMessagesIndexView: PostboxView {
+    public let peer: Peer?
+    public let items: [EngineMessageHistorySavedMessagesThread.Item]
+    public let isLoading: Bool
+    
+    init(_ view: MutableMessageHistorySavedMessagesIndexView) {
+        self.peer = view.peer
+        
+        var items: [EngineMessageHistorySavedMessagesThread.Item] = []
+        for item in view.items {
+            items.append(EngineMessageHistorySavedMessagesThread.Item(
+                id: item.id,
+                peer: item.peer,
+                pinnedIndex: item.pinnedIndex,
+                index: item.index,
+                topMessage: item.topMessage,
+                unreadCount: item.unreadCount,
+                markedUnread: item.markedUnread,
+                embeddedInterfaceState: item.embeddedInterfaceState
+            ))
+        }
+        self.items = items
+        
+        self.isLoading = view.isLoading
+    }
+}

submodules/Postbox/Sources/MessageHistorySavedMessagesStatsView.swift

@@ -0,0 +1,55 @@
+import Foundation
+
+final class MutableMessageHistorySavedMessagesStatsView: MutablePostboxView {
+    fileprivate let peerId: PeerId
+    fileprivate var count: Int = 0
+    fileprivate var isLoading: Bool = false
+    
+    init(postbox: PostboxImpl, peerId: PeerId) {
+        self.peerId = peerId
+        
+        self.reload(postbox: postbox)
+    }
+    
+    private func reload(postbox: PostboxImpl) {
+        let validIndexBoundary = postbox.peerThreadCombinedStateTable.get(peerId: peerId)?.validIndexBoundary
+        self.isLoading = validIndexBoundary == nil
+        
+        if !self.isLoading {
+            self.count = postbox.messageHistoryThreadIndexTable.getCount(peerId: self.peerId)
+        } else {
+            self.count = 0
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if transaction.updatedMessageThreadPeerIds.contains(self.peerId) {
+            self.reload(postbox: postbox)
+            updated = true
+        }
+        
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        self.reload(postbox: postbox)
+        
+        return true
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageHistorySavedMessagesStatsView(self)
+    }
+}
+
+public final class MessageHistorySavedMessagesStatsView: PostboxView {
+    public let isLoading: Bool
+    public let count: Int
+    
+    init(_ view: MutableMessageHistorySavedMessagesStatsView) {
+        self.isLoading = view.isLoading
+        self.count = view.count
+    }
+}

submodules/Postbox/Sources/MessageHistorySynchronizeReadStateTable.swift

@@ -0,0 +1,90 @@
+import Foundation
+
+public enum PeerReadStateSynchronizationOperation: Equatable {
+    case Push(state: CombinedPeerReadState?, thenSync: Bool)
+    case Validate
+}
+
+final class MessageHistorySynchronizeReadStateTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .int64, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8)
+    
+    private func key(peerId: PeerId) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: peerId.toInt64())
+        return self.sharedKey
+    }
+    
+    private var updatedPeerIds: [PeerId: PeerReadStateSynchronizationOperation?] = [:]
+    
+    private func lowerBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: 0)
+        return key
+    }
+    
+    private func upperBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        memset(key.memory, 0xff, key.length)
+        return key
+    }
+    
+    func set(_ peerId: PeerId, operation: PeerReadStateSynchronizationOperation?, operations: inout [PeerId: PeerReadStateSynchronizationOperation?]) {
+        self.updatedPeerIds[peerId] = operation
+        operations[peerId] = operation
+    }
+    
+    func get(getCombinedPeerReadState: (PeerId) -> CombinedPeerReadState?) -> [PeerId: PeerReadStateSynchronizationOperation] {
+        self.beforeCommit()
+        
+        var operations: [PeerId: PeerReadStateSynchronizationOperation] = [:]
+        self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), values: { key, value in
+            let peerId = PeerId(key.getInt64(0))
+            var operationValue: Int8 = 0
+            value.read(&operationValue, offset: 0, length: 1)
+            
+            let operation: PeerReadStateSynchronizationOperation
+            if operationValue == 0 {
+                var syncValue: Int8 = 0
+                value.read(&syncValue, offset: 0, length: 1)
+                operation = .Push(state: getCombinedPeerReadState(peerId), thenSync: syncValue != 0)
+            } else {
+                operation = .Validate
+            }
+            
+            operations[peerId] = operation
+            return true
+        }, limit: 0)
+        return operations
+    }
+    
+    override func beforeCommit() {
+        if !self.updatedPeerIds.isEmpty {
+            let key = ValueBoxKey(length: 8)
+            let buffer = WriteBuffer()
+            for (peerId, operation) in self.updatedPeerIds {
+                key.setInt64(0, value: peerId.toInt64())
+                if let operation = operation {
+                    buffer.reset()
+                    switch operation {
+                        case let .Push(_, thenSync):
+                            var operationValue: Int8 = 0
+                            buffer.write(&operationValue, offset: 0, length: 1)
+                            var syncValue: Int8 = thenSync ? 1 : 0
+                            buffer.write(&syncValue, offset: 0, length: 1)
+                        case .Validate:
+                            var operationValue: Int8 = 1
+                            buffer.write(&operationValue, offset: 0, length: 1)
+                    }
+                    
+                    self.valueBox.set(self.table, key: key, value: buffer)
+                } else {
+                    self.valueBox.remove(self.table, key: key, secure: false)
+                }
+            }
+            self.updatedPeerIds.removeAll()
+        }
+    }
+}

submodules/Postbox/Sources/MessageHistoryTagSummaryView.swift

@@ -0,0 +1,48 @@
+import Foundation
+
+final class MutableMessageHistoryTagSummaryView: MutablePostboxView {
+    private let tag: MessageTags
+    private let peerId: PeerId
+    private let threadId: Int64?
+    private let namespace: MessageId.Namespace
+    private let customTag: MemoryBuffer?
+    
+    fileprivate var count: Int32?
+    
+    init(postbox: PostboxImpl, tag: MessageTags, peerId: PeerId, threadId: Int64?, namespace: MessageId.Namespace, customTag: MemoryBuffer?) {
+        self.tag = tag
+        self.peerId = peerId
+        self.threadId = threadId
+        self.namespace = namespace
+        self.customTag = customTag
+        
+        self.count = postbox.messageHistoryTagsSummaryTable.get(MessageHistoryTagsSummaryKey(tag: tag, peerId: peerId, threadId: threadId, namespace: namespace, customTag: customTag))?.count
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var hasChanges = false
+        
+        if let summary = transaction.currentUpdatedMessageTagSummaries[MessageHistoryTagsSummaryKey(tag: self.tag, peerId: self.peerId, threadId: self.threadId, namespace: self.namespace, customTag: self.customTag)] {
+            self.count = summary.count
+            hasChanges = true
+        }
+        
+        return hasChanges
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageHistoryTagSummaryView(self)
+    }
+}
+
+public final class MessageHistoryTagSummaryView: PostboxView {
+    public let count: Int32?
+    
+    init(_ view: MutableMessageHistoryTagSummaryView) {
+        self.count = view.count
+    }
+}

submodules/Postbox/Sources/MessageHistoryTagsSummaryTable.swift

@@ -0,0 +1,277 @@
+import Foundation
+
+public struct MessageHistoryTagNamespaceCountValidityRange: Equatable {
+    public let maxId: MessageId.Id
+    
+    public init(maxId: MessageId.Id) {
+        self.maxId = maxId
+    }
+    
+    public static func ==(lhs: MessageHistoryTagNamespaceCountValidityRange, rhs: MessageHistoryTagNamespaceCountValidityRange) -> Bool {
+        return lhs.maxId == rhs.maxId
+    }
+    
+    public func contains(_ id: MessageId.Id) -> Bool {
+        return id <= self.maxId
+    }
+}
+
+public struct MessageHistoryTagNamespaceSummary: Equatable, CustomStringConvertible {
+    public let version: Int32
+    public let count: Int32
+    public let range: MessageHistoryTagNamespaceCountValidityRange
+    
+    public init(version: Int32, count: Int32, range: MessageHistoryTagNamespaceCountValidityRange) {
+        self.version = version
+        self.count = count
+        self.range = range
+    }
+    
+    public static func ==(lhs: MessageHistoryTagNamespaceSummary, rhs: MessageHistoryTagNamespaceSummary) -> Bool {
+        return lhs.version == rhs.version && lhs.count == rhs.count && lhs.range == rhs.range
+    }
+    
+    func withAddedCount(_ value: Int32) -> MessageHistoryTagNamespaceSummary {
+        return MessageHistoryTagNamespaceSummary(version: self.version, count: Int32(clamping: Int64(self.count) + Int64(value)), range: self.range)
+    }
+    
+    public var description: String {
+        return "(version: \(self.version), count: \(self.count), range: (maxId: \(self.range.maxId)))"
+    }
+}
+
+struct MessageHistoryTagsSummaryKey: Equatable, Hashable {
+    let tag: MessageTags
+    let peerId: PeerId
+    let threadId: Int64?
+    let namespace: MessageId.Namespace
+    let customTag: MemoryBuffer?
+    
+    init(tag: MessageTags, peerId: PeerId, threadId: Int64?, namespace: MessageId.Namespace, customTag: MemoryBuffer?) {
+        self.tag = tag
+        self.peerId = peerId
+        self.threadId = threadId
+        self.namespace = namespace
+        self.customTag = customTag
+    }
+}
+
+private func readSummary(_ value: ReadBuffer) -> MessageHistoryTagNamespaceSummary {
+    var versionValue: Int32 = 0
+    value.read(&versionValue, offset: 0, length: 4)
+    var countValue: Int32 = 0
+    value.read(&countValue, offset: 0, length: 4)
+    var maxIdValue: Int32 = 0
+    value.read(&maxIdValue, offset: 0, length: 4)
+
+    return MessageHistoryTagNamespaceSummary(version: versionValue, count: countValue, range: MessageHistoryTagNamespaceCountValidityRange(maxId: maxIdValue))
+}
+
+private func writeSummary(_ summary: MessageHistoryTagNamespaceSummary, to buffer: WriteBuffer) {
+    var versionValue: Int32 = summary.version
+    buffer.write(&versionValue, offset: 0, length: 4)
+    var countValue: Int32 = summary.count
+    buffer.write(&countValue, offset: 0, length: 4)
+    var maxIdValue: Int32 = summary.range.maxId
+    buffer.write(&maxIdValue, offset: 0, length: 4)
+}
+
+private struct CachedEntry {
+    let summary: MessageHistoryTagNamespaceSummary?
+}
+
+class MessageHistoryTagsSummaryTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let invalidateTable: InvalidatedMessageHistoryTagsSummaryTable
+    
+    private var cachedSummaries: [MessageHistoryTagsSummaryKey: CachedEntry] = [:]
+    private var updatedKeys = Set<MessageHistoryTagsSummaryKey>()
+    
+    private let sharedSimpleKey = ValueBoxKey(length: 4 + 8 + 4)
+    private let sharedThreadKey = ValueBoxKey(length: 4 + 8 + 4 + 8)
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, invalidateTable: InvalidatedMessageHistoryTagsSummaryTable) {
+        self.invalidateTable = invalidateTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func keyShared(key: MessageHistoryTagsSummaryKey) -> ValueBoxKey {
+        return self.keyInternal(key: key, allowShared: true)
+    }
+    
+    private func keyInternal(key: MessageHistoryTagsSummaryKey, allowShared: Bool) -> ValueBoxKey {
+        if let customTag = key.customTag {
+            if customTag.length != 10 && customTag.length != 7 {
+                assert(true)
+            }
+            
+            var keyLength = 4 + 8 + 4
+            keyLength += 8
+            keyLength += customTag.length
+            let result = ValueBoxKey(length: keyLength)
+            
+            var offset = 0
+            result.setUInt32(offset, value: key.tag.rawValue)
+            offset += 4
+            
+            result.setInt64(offset, value: key.peerId.toInt64())
+            offset += 8
+            
+            result.setInt32(offset, value: key.namespace)
+            offset += 4
+            
+            result.setInt64(offset, value: key.threadId ?? 0)
+            offset += 8
+            
+            customTag.withRawBufferPointer { buffer in
+                result.setBytes(offset, value: buffer)
+                offset += buffer.count
+            }
+            
+            return result
+        } else if let threadId = key.threadId {
+            let result: ValueBoxKey
+            if allowShared {
+                result = self.sharedThreadKey
+            } else {
+                result = ValueBoxKey(length: 4 + 8 + 4 + 8)
+            }
+            result.setUInt32(0, value: key.tag.rawValue)
+            result.setInt64(4, value: key.peerId.toInt64())
+            result.setInt32(4 + 8, value: key.namespace)
+            result.setInt64(4 + 8 + 4, value: threadId)
+            return result
+        } else {
+            let result: ValueBoxKey
+            if allowShared {
+                result = self.sharedSimpleKey
+            } else {
+                result = ValueBoxKey(length: 4 + 8 + 4)
+            }
+            result.setUInt32(0, value: key.tag.rawValue)
+            result.setInt64(4, value: key.peerId.toInt64())
+            result.setInt32(4 + 8, value: key.namespace)
+            return result
+        }
+    }
+    
+    func get(_ key: MessageHistoryTagsSummaryKey) -> MessageHistoryTagNamespaceSummary? {
+        if let cached = self.cachedSummaries[key] {
+            return cached.summary
+        } else if let value = self.valueBox.get(self.table, key: self.keyShared(key: key)) {
+            let entry = readSummary(value)
+            self.cachedSummaries[key] = CachedEntry(summary: entry)
+            return entry
+        } else {
+            self.cachedSummaries[key] = CachedEntry(summary: nil)
+            return nil
+        }
+    }
+    
+    func getCustomTags(tag: MessageTags, peerId: PeerId, threadId: Int64?, namespace: MessageId.Namespace) -> [MemoryBuffer] {
+        let key = MessageHistoryTagsSummaryKey(tag: tag, peerId: peerId, threadId: threadId, namespace: namespace, customTag: nil)
+        
+        let peerKey = self.keyInternal(key: key, allowShared: false)
+        let prefixLength = 4 + 8 + 4 + 8
+        var result: [MemoryBuffer] = []
+        self.valueBox.range(self.table, start: peerKey.predecessor, end: peerKey.successor, keys: { key in
+            let testPeerId = key.getInt64(4)
+            assert(PeerId(testPeerId) == peerId)
+            let testNamespace = key.getInt32(4 + 8)
+            assert(testNamespace == namespace)
+            
+            if key.length > prefixLength {
+                result.append(key.getMemoryBuffer(prefixLength, length: key.length - prefixLength))
+            }
+            return true
+        }, limit: 0)
+        
+        for updatedKey in self.updatedKeys {
+            if updatedKey.peerId == peerId && updatedKey.tag == tag && updatedKey.threadId == threadId && updatedKey.namespace == namespace {
+                if let customTag = updatedKey.customTag {
+                    if !result.contains(customTag) {
+                        result.append(customTag)
+                    }
+                }
+            }
+        }
+        
+        return result
+    }
+    
+    private func set(_ key: MessageHistoryTagsSummaryKey, summary: MessageHistoryTagNamespaceSummary, updatedSummaries: inout [MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary]) {
+        if self.get(key) != summary {
+            if key.tag.rawValue == 2048 {
+                postboxLog("[MessageHistoryTagsSummaryTable] set \(key.tag.rawValue) for \(key.peerId) to \(summary.count)")
+            }
+            self.updatedKeys.insert(key)
+            self.cachedSummaries[key] = CachedEntry(summary: summary)
+            updatedSummaries[key] = summary
+        }
+    }
+    
+    func addMessage(key: MessageHistoryTagsSummaryKey, id: MessageId.Id, isNewlyAdded: Bool, updatedSummaries: inout [MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        if let current = self.get(key) {
+            if !isNewlyAdded || !current.range.contains(id) {
+                self.set(key, summary: current.withAddedCount(1), updatedSummaries: &updatedSummaries)
+                if current.range.maxId == 0 {
+                    self.invalidateTable.insert(InvalidatedMessageHistoryTagsSummaryKey(peerId: key.peerId, namespace: key.namespace, tagMask: key.tag, threadId: key.threadId, customTag: key.customTag), operations: &invalidateSummaries)
+                }
+            }
+        } else {
+            self.set(key, summary: MessageHistoryTagNamespaceSummary(version: 0, count: 1, range: MessageHistoryTagNamespaceCountValidityRange(maxId: 0)), updatedSummaries: &updatedSummaries)
+            self.invalidateTable.insert(InvalidatedMessageHistoryTagsSummaryKey(peerId: key.peerId, namespace: key.namespace, tagMask: key.tag, threadId: key.threadId, customTag: key.customTag), operations: &invalidateSummaries)
+        }
+    }
+    
+    func removeMessage(key: MessageHistoryTagsSummaryKey, id: MessageId.Id, updatedSummaries: inout [MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        if let current = self.get(key) {
+            if current.count == 0 {
+            } else {
+                self.set(key, summary: current.withAddedCount(-1), updatedSummaries: &updatedSummaries)
+            }
+        }
+    }
+    
+    func replace(key: MessageHistoryTagsSummaryKey, count: Int32, maxId: MessageId.Id, updatedSummaries: inout [MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary]) {
+        var version: Int32 = 0
+        if let current = self.get(key) {
+            version = current.version + 1
+        }
+        self.set(key, summary: MessageHistoryTagNamespaceSummary(version: version, count: count, range: MessageHistoryTagNamespaceCountValidityRange(maxId: maxId)), updatedSummaries: &updatedSummaries)
+    }
+    
+    override func clearMemoryCache() {
+        self.cachedSummaries.removeAll()
+        assert(self.updatedKeys.isEmpty)
+    }
+    
+    override func beforeCommit() {
+        if !self.updatedKeys.isEmpty {
+            let buffer = WriteBuffer()
+            for key in self.updatedKeys {
+                if let cached = self.cachedSummaries[key] {
+                    if let summary = cached.summary {
+                        buffer.reset()
+                        writeSummary(summary, to: buffer)
+                        self.valueBox.set(self.table, key: self.keyShared(key: key), value: buffer)
+                    } else {
+                        assertionFailure()
+                        self.valueBox.remove(self.table, key: self.keyShared(key: key), secure: false)
+                    }
+                } else {
+                    assertionFailure()
+                }
+            }
+            self.updatedKeys.removeAll()
+        }
+
+        if !self.useCaches {
+            self.cachedSummaries.removeAll()
+        }
+    }
+}

submodules/Postbox/Sources/MessageHistoryTagsTable.swift

@@ -0,0 +1,196 @@
+import Foundation
+
+private func extractKey(_ key: ValueBoxKey) -> MessageIndex {
+    return MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 4), id: key.getInt32(8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 4 + 4))
+}
+
+class MessageHistoryTagsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4)
+    
+    private let summaryTable: MessageHistoryTagsSummaryTable
+    private let summaryTags: MessageTags
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, seedConfiguration: SeedConfiguration, summaryTable: MessageHistoryTagsSummaryTable) {
+        self.summaryTable = summaryTable
+        self.summaryTags = seedConfiguration.messageTagsWithSummary
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(tag: MessageTags, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey {
+        key.setInt64(0, value: index.id.peerId.toInt64())
+        key.setUInt32(8, value: tag.rawValue)
+        key.setInt32(8 + 4, value: index.id.namespace)
+        key.setInt32(8 + 4 + 4, value: index.timestamp)
+        key.setInt32(8 + 4 + 4 + 4, value: index.id.id)
+        return key
+    }
+    
+    private func lowerBound(tag: MessageTags, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setUInt32(8, value: tag.rawValue)
+        key.setInt32(8 + 4, value: namespace)
+        return key
+    }
+    
+    private func upperBound(tag: MessageTags, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        return self.lowerBound(tag: tag, peerId: peerId, namespace: namespace).successor
+    }
+    
+    func add(tags: MessageTags, index: MessageIndex, isNewlyAdded: Bool, updatedSummaries: inout[MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        for tag in tags {
+            self.valueBox.set(self.table, key: self.key(tag: tag, index: index, key: self.sharedKey), value: MemoryBuffer())
+            if self.summaryTags.contains(tag) {
+                self.summaryTable.addMessage(key: MessageHistoryTagsSummaryKey(tag: tag, peerId: index.id.peerId, threadId: nil, namespace: index.id.namespace, customTag: nil), id: index.id.id, isNewlyAdded: isNewlyAdded, updatedSummaries: &updatedSummaries, invalidateSummaries: &invalidateSummaries)
+            }
+        }
+    }
+    
+    func remove(tags: MessageTags, index: MessageIndex, updatedSummaries: inout[MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        for tag in tags {
+            self.valueBox.remove(self.table, key: self.key(tag: tag, index: index, key: self.sharedKey), secure: false)
+            
+            if self.summaryTags.contains(tag) {
+                self.summaryTable.removeMessage(key: MessageHistoryTagsSummaryKey(tag: tag, peerId: index.id.peerId, threadId: nil, namespace: index.id.namespace, customTag: nil), id: index.id.id, updatedSummaries: &updatedSummaries, invalidateSummaries: &invalidateSummaries)
+            }
+        }
+    }
+    
+    func entryExists(tag: MessageTags, index: MessageIndex) -> Bool {
+        return self.valueBox.exists(self.table, key: self.key(tag: tag, index: index, key: self.sharedKey))
+    }
+    
+    func entryLocation(at index: MessageIndex, tag: MessageTags) -> MessageHistoryEntryLocation? {
+        if let _ = self.valueBox.get(self.table, key: self.key(tag: tag, index: index)) {
+            var greaterCount = 0
+            self.valueBox.range(self.table, start: self.key(tag: tag, index: index), end: self.upperBound(tag: tag, peerId: index.id.peerId, namespace: index.id.namespace), keys: { _ in
+                greaterCount += 1
+                return true
+            }, limit: 0)
+            
+            var lowerCount = 0
+            self.valueBox.range(self.table, start: self.key(tag: tag, index: index), end: self.lowerBound(tag: tag, peerId: index.id.peerId, namespace: index.id.namespace), keys: { _ in
+                lowerCount += 1
+                return true
+            }, limit: 0)
+            
+            return MessageHistoryEntryLocation(index: lowerCount, count: greaterCount + lowerCount + 1)
+        }
+        return nil
+    }
+    
+    func earlierIndices(tag: MessageTags, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, minIndex: MessageIndex? = nil, count: Int) -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(tag: tag, index: index).successor
+            } else {
+                key = self.key(tag: tag, index: index)
+            }
+        } else {
+            key = self.upperBound(tag: tag, peerId: peerId, namespace: namespace)
+        }
+        let endKey: ValueBoxKey
+        if let minIndex = minIndex {
+            endKey = self.key(tag: tag, index: minIndex)
+        } else {
+            endKey = self.lowerBound(tag: tag, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: endKey, keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func laterIndices(tag: MessageTags, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, count: Int) -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(tag: tag, index: index).predecessor
+            } else {
+                key = self.key(tag: tag, index: index)
+            }
+        } else {
+            key = self.lowerBound(tag: tag, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: self.upperBound(tag: tag, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func getMessageCountInRange(tag: MessageTags, peerId: PeerId, namespace: MessageId.Namespace, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int {
+        precondition(lowerBound.id.namespace == namespace)
+        precondition(upperBound.id.namespace == namespace)
+        var lowerBoundKey = self.key(tag: tag, index: lowerBound)
+        if lowerBound.timestamp > 1 {
+            lowerBoundKey = lowerBoundKey.predecessor
+        }
+        var upperBoundKey = self.key(tag: tag, index: upperBound)
+        if upperBound.timestamp < Int32.max - 1 {
+            upperBoundKey = upperBoundKey.successor
+        }
+        return Int(self.valueBox.count(self.table, start: lowerBoundKey, end: upperBoundKey))
+    }
+    
+    func latestIndex(tag: MessageTags, peerId: PeerId, namespace: MessageId.Namespace) -> MessageIndex? {
+        var result: MessageIndex?
+        self.valueBox.range(self.table, start: self.lowerBound(tag: tag, peerId: peerId, namespace: namespace), end: self.upperBound(tag: tag, peerId: peerId, namespace: namespace), keys: { key in
+            result = extractKey(key)
+            return true
+        }, limit: 1)
+        return result
+    }
+    
+    func findRandomIndex(peerId: PeerId, namespace: MessageId.Namespace, tag: MessageTags, ignoreIds: ([MessageId], Set<MessageId>), isMessage: (MessageIndex) -> Bool) -> MessageIndex? {
+        var indices: [MessageIndex] = []
+        self.valueBox.range(self.table, start: self.lowerBound(tag: tag, peerId: peerId, namespace: namespace), end: self.upperBound(tag: tag, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: 0)
+        var checkedIndices = Set<Int>()
+        while checkedIndices.count < indices.count {
+            let i = Int(arc4random_uniform(UInt32(indices.count)))
+            if checkedIndices.contains(i) {
+                continue
+            }
+            checkedIndices.insert(i)
+            let index = indices[i]
+            if isMessage(index) && !ignoreIds.1.contains(index.id) {
+                return index
+            }
+        }
+        checkedIndices.removeAll()
+        let lastId = ignoreIds.0.last
+        while checkedIndices.count < indices.count {
+            let i = Int(arc4random_uniform(UInt32(indices.count)))
+            if checkedIndices.contains(i) {
+                continue
+            }
+            checkedIndices.insert(i)
+            let index = indices[i]
+            if isMessage(index) && lastId != index.id {
+                return index
+            }
+        }
+        return nil
+    }
+    
+    func debugGetAllIndices() -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        self.valueBox.scan(self.table, values: { key, value in
+            indices.append(extractKey(key))
+            return true
+        })
+        return indices
+    }
+}

submodules/Postbox/Sources/MessageHistoryTextIndexTable.swift

@@ -0,0 +1,99 @@
+import Foundation
+
+private func collectionId(_ peerId: PeerId) -> String {
+    return "p\(UInt64(bitPattern: peerId.toInt64()))"
+}
+
+private func itemId(_ messageId: MessageId) -> String {
+    return "p\(UInt64(bitPattern: messageId.peerId.toInt64()))a\(UInt32(bitPattern: messageId.namespace))b\(UInt32(bitPattern: messageId.id))"
+}
+
+private func messageTags(_ tags: MessageTags) -> String {
+    var result = ""
+    for tag in tags {
+        if !result.isEmpty {
+            result += " "
+        }
+        result += "t\(tag.rawValue)"
+    }
+    return result
+}
+
+private func parseMessageId(_ value: String) -> MessageId? {
+    if !value.hasPrefix("p") {
+        return nil
+    }
+    guard let aRange = value.range(of: "a") else {
+        return nil
+    }
+    guard let bRange = value.range(of: "b") else {
+        return nil
+    }
+    let pString = value[value.index(value.startIndex, offsetBy: 1) ..< aRange.lowerBound]
+    let nString = value[aRange.upperBound ..< bRange.lowerBound]
+    let iString = value[bRange.upperBound ..< value.endIndex]
+    
+    guard let pValue = UInt64(pString) else {
+        return nil
+    }
+    guard let nValue = UInt32(nString) else {
+        return nil
+    }
+    guard let iValue = UInt32(iString) else {
+        return nil
+    }
+    
+    return MessageId(peerId: PeerId(Int64(bitPattern: pValue)), namespace: Int32(bitPattern: nValue), id: Int32(bitPattern: iValue))
+}
+
+private let alphanumerics = CharacterSet.alphanumerics
+
+final class MessageHistoryTextIndexTable {
+    static func tableSpec(_ id: Int32) -> ValueBoxFullTextTable {
+        return ValueBoxFullTextTable(id: id)
+    }
+    
+    private let valueBox: ValueBox
+    private let table: ValueBoxFullTextTable
+    
+    init(valueBox: ValueBox, table: ValueBoxFullTextTable) {
+        self.valueBox = valueBox
+        self.table = table
+    }
+    
+    func add(messageId: MessageId, text: String, tags: MessageTags) {
+        self.valueBox.fullTextSet(self.table, collectionId: collectionId(messageId.peerId), itemId: itemId(messageId), contents: text, tags: messageTags(tags))
+    }
+    
+    func remove(messageId: MessageId) {
+        self.valueBox.fullTextRemove(self.table, itemId: itemId(messageId), secure: true)
+    }
+    
+    func search(peerId: PeerId?, text: String, tags: MessageTags?) -> [MessageId] {
+        var escapedText = String(text.map({ c in
+            var codeUnits: [UnicodeScalar] = []
+            for codeUnit in String(c).unicodeScalars {
+                codeUnits.append(codeUnit)
+            }
+            for codeUnit in codeUnits {
+                if !alphanumerics.contains(codeUnit) {
+                    return " "
+                }
+            }
+            return c
+        }))
+        if !escapedText.isEmpty {
+            escapedText += "*"
+        }
+        var result: [MessageId] = []
+        self.valueBox.fullTextMatch(self.table, collectionId: peerId.flatMap { collectionId($0) }, query: escapedText, tags: tags.flatMap(messageTags), values: { _, itemId in
+            if let messageId = parseMessageId(itemId) {
+                result.append(messageId)
+            } else {
+                assertionFailure()
+            }
+            return true
+        })
+        return result
+    }
+}

submodules/Postbox/Sources/MessageHistoryThreadHoleIndexTable.swift

@@ -0,0 +1,436 @@
+import Foundation
+
+private func decomposeKey(_ key: ValueBoxKey) -> (threadId: Int64, id: MessageId, space: MessageHistoryHoleSpace) {
+    let tag = MessageTags(rawValue: key.getUInt32(8 + 8 + 4))
+    let space: MessageHistoryHoleSpace
+    if tag.rawValue == 0 {
+        space = .everywhere
+    } else {
+        space = .tag(tag)
+    }
+    return (key.getInt64(8), MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 8), id: key.getInt32(8 + 8 + 4 + 4)), space)
+}
+
+private func decodeValue(value: ReadBuffer, peerId: PeerId, namespace: MessageId.Namespace) -> MessageId {
+    var id: Int32 = 0
+    value.read(&id, offset: 0, length: 4)
+    return MessageId(peerId: peerId, namespace: namespace, id: id)
+}
+
+final class MessageHistoryThreadHoleIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    let metadataTable: MessageHistoryMetadataTable
+    let seedConfiguration: SeedConfiguration
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, metadataTable: MessageHistoryMetadataTable, seedConfiguration: SeedConfiguration) {
+        self.seedConfiguration = seedConfiguration
+        self.metadataTable = metadataTable
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(threadId: Int64, id: MessageId, space: MessageHistoryHoleSpace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4)
+        key.setInt64(0, value: id.peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: id.namespace)
+        let tagValue: UInt32
+        switch space {
+            case .everywhere:
+                tagValue = 0
+            case let .tag(tag):
+                tagValue = tag.rawValue
+        }
+        key.setUInt32(8 + 8 + 4, value: tagValue)
+        key.setInt32(8 + 8 + 4 + 4, value: id.id)
+        return key
+    }
+    
+    private func lowerBound(peerId: PeerId, threadId: Int64) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, threadId: Int64) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId, threadId: threadId).successor
+    }
+    
+    private func lowerBound(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: namespace)
+        let tagValue: UInt32
+        switch space {
+            case .everywhere:
+                tagValue = 0
+            case let .tag(tag):
+                tagValue = tag.rawValue
+        }
+        key.setUInt32(8 + 8 + 4, value: tagValue)
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: namespace)
+        let tagValue: UInt32
+        switch space {
+            case .everywhere:
+                tagValue = 0
+            case let .tag(tag):
+                tagValue = tag.rawValue
+        }
+        key.setUInt32(8 + 8 + 4, value: tagValue)
+        return key.successor
+    }
+    
+    private func namespaceLowerBound(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: namespace)
+        return key
+    }
+    
+    private func namespaceUpperBound(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: namespace)
+        return key.successor
+    }
+    
+    private func ensureInitialized(peerId: PeerId, threadId: Int64) {
+        if !self.metadataTable.isThreadHoleIndexInitialized(peerId: peerId, threadId: threadId) {
+            postboxLog("MessageHistoryThreadHoleIndexTable: Initializing \(peerId) \(threadId)")
+            self.metadataTable.setIsThreadHoleIndexInitialized(peerId: peerId, threadId: threadId)
+            
+            if let messageNamespaces = self.seedConfiguration.messageThreadHoles(peerId.namespace, threadId) {
+                for namespace in messageNamespaces {
+                    var operations: [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]] = [:]
+                    self.add(peerId: peerId, threadId: threadId, namespace: namespace, space: .everywhere, range: 1 ... (Int32.max - 1), operations: &operations)
+                }
+            }
+        }
+    }
+    
+    func existingNamespaces(peerId: PeerId, threadId: Int64, holeSpace: MessageHistoryHoleSpace) -> Set<MessageId.Namespace> {
+        self.ensureInitialized(peerId: peerId, threadId: threadId)
+        
+        var result = Set<MessageId.Namespace>()
+        var currentLowerBound = self.lowerBound(peerId: peerId, threadId: threadId)
+        let upperBound = self.upperBound(peerId: peerId, threadId: threadId)
+        while true {
+            var idAndSpace: (Int64, MessageId, MessageHistoryHoleSpace)?
+            self.valueBox.range(self.table, start: currentLowerBound, end: upperBound, keys: { key in
+                idAndSpace = decomposeKey(key)
+                return false
+            }, limit: 1)
+            if let (_, id, space) = idAndSpace {
+                if space == holeSpace {
+                    result.insert(id.namespace)
+                }
+                currentLowerBound = self.upperBound(peerId: peerId, threadId: threadId, namespace: id.namespace, space: space)
+            } else {
+                break
+            }
+        }
+        return result
+    }
+    
+    private func scanSpaces(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace) -> [MessageHistoryHoleSpace] {
+        self.ensureInitialized(peerId: peerId, threadId: threadId)
+        
+        var currentLowerBound = self.namespaceLowerBound(peerId: peerId, threadId: threadId, namespace: namespace)
+        var result: [MessageHistoryHoleSpace] = []
+        while true {
+            var found = false
+            self.valueBox.range(self.table, start: currentLowerBound, end: self.namespaceUpperBound(peerId: peerId, threadId: threadId, namespace: namespace), keys: { key in
+                let space = decomposeKey(key).space
+                result.append(space)
+                currentLowerBound = self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space)
+                found = true
+                return false
+            }, limit: 1)
+            if !found {
+                break
+            }
+        }
+        assert(Set(result).count == result.count)
+        return result
+    }
+    
+    func containing(threadId: Int64, id: MessageId) -> [MessageHistoryHoleSpace: ClosedRange<MessageId.Id>] {
+        self.ensureInitialized(peerId: id.peerId, threadId: threadId)
+        
+        var result: [MessageHistoryHoleSpace: ClosedRange<MessageId.Id>] = [:]
+        for space in self.scanSpaces(peerId: id.peerId, threadId: threadId, namespace: id.namespace) {
+            self.valueBox.range(self.table, start: self.key(threadId: threadId, id: id, space: space), end: self.upperBound(peerId: id.peerId, threadId: threadId, namespace: id.namespace, space: space), values: { key, value in
+                let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+                assert(keyThreadId == threadId)
+                assert(keySpace == space)
+                assert(upperId.peerId == id.peerId)
+                assert(upperId.namespace == id.namespace)
+                let lowerId = decodeValue(value: value, peerId: id.peerId, namespace: id.namespace)
+                let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+                result[space] = holeRange
+                return false
+            }, limit: 1)
+        }
+        return result
+    }
+    
+    func latest(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> ClosedRange<MessageId.Id>? {
+        self.ensureInitialized(peerId: peerId, threadId: threadId)
+        
+        var result: ClosedRange<MessageId.Id>?
+        self.valueBox.range(self.table, start: self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), end: self.lowerBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+            
+            let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            result = lowerId.id ... upperId.id
+            
+            return false
+        }, limit: 1)
+        
+        return result
+    }
+    
+    func closest(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>) -> IndexSet {
+        self.ensureInitialized(peerId: peerId, threadId: threadId)
+        
+        var result = IndexSet()
+        
+        func processIntersectingRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if holeRange.overlaps(range) {
+                result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+            }
+        }
+        
+        func processEdgeRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            result.insert(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+        }
+        
+        self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), space: space).predecessor, end: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space).successor, values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space), end: self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+            processIntersectingRange(key, value)
+            return true
+        }, limit: 1)
+        
+        if !result.contains(Int(range.lowerBound)) {
+            self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.lowerBound), space: space), end: self.lowerBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        if !result.contains(Int(range.upperBound)) {
+            self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space), end: self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+                processEdgeRange(key, value)
+                return true
+            }, limit: 1)
+        }
+        
+        return result
+    }
+    
+    func add(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        self.ensureInitialized(peerId: peerId, threadId: threadId)
+        
+        self.addInternal(peerId: peerId, threadId: threadId, namespace: namespace, space: space, range: range, operations: &operations)
+        
+        /*switch space {
+            case .everywhere:
+                if let namespaceHoleTags = self.seedConfiguration.messageHoles[peerId.namespace]?[namespace] {
+                    for tag in namespaceHoleTags {
+                        self.addInternal(peerId: peerId, threadId: threadId, namespace: namespace, space: .tag(tag), range: range, operations: &operations)
+                    }
+                }
+            case .tag:
+                break
+        }*/
+    }
+    
+    private func addInternal(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        let clippedLowerBound = max(1, range.lowerBound)
+        let clippedUpperBound = min(Int32.max - 1, range.upperBound)
+        if clippedLowerBound > clippedUpperBound {
+            return
+        }
+        let clippedRange = clippedLowerBound ... clippedUpperBound
+        
+        var insertedIndices = IndexSet()
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        var alreadyMapped = false
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<Int32> = lowerId.id ... upperId.id
+            if clippedRange.lowerBound >= holeRange.lowerBound && clippedRange.upperBound <= holeRange.upperBound {
+                alreadyMapped = true
+                return
+            } else if clippedRange.overlaps(holeRange) || (holeRange.upperBound != Int32.max && clippedRange.lowerBound == holeRange.upperBound + 1) || clippedRange.upperBound == holeRange.lowerBound - 1 {
+                removeKeys.append(upperId.id)
+                let unionRange: ClosedRange = min(clippedRange.lowerBound, holeRange.lowerBound) ... max(clippedRange.upperBound, holeRange.upperBound)
+                insertRanges.insert(integersIn: Int(unionRange.lowerBound) ... Int(unionRange.upperBound))
+            }
+        }
+        
+        let lowerScanBound = max(0, clippedRange.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), space: space), end: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), space: space).successor, values: { key, value in
+            processRange(key, value)
+            if alreadyMapped {
+                return false
+            }
+            return true
+        }, limit: 0)
+        
+        if !alreadyMapped {
+            self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: clippedRange.upperBound), space: space), end: self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+                processRange(key, value)
+                if alreadyMapped {
+                    return false
+                }
+                return true
+            }, limit: 1)
+        }
+        
+        if alreadyMapped {
+            return
+        }
+        
+        insertRanges.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        insertedIndices.insert(integersIn: Int(clippedRange.lowerBound) ... Int(clippedRange.upperBound))
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: id), space: space), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), space: space), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        addMessageHistoryHoleOperation(.insert(clippedRange), peerId: peerId, threadId: threadId, namespace: namespace, space: MessageHistoryHoleOperationSpace(space), to: &operations)
+    }
+    
+    func remove(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        self.ensureInitialized(peerId: peerId, threadId: threadId)
+        
+        self.removeInternal(peerId: peerId, threadId: threadId, namespace: namespace, space: space, range: range, operations: &operations)
+        
+        switch space {
+            case .everywhere:
+                if let namespaceHoleTags = self.seedConfiguration.messageHoles[peerId.namespace]?[namespace] {
+                    for tag in namespaceHoleTags {
+                        self.removeInternal(peerId: peerId, threadId: threadId, namespace: namespace, space: .tag(tag), range: range, operations: &operations)
+                    }
+                }
+            case .tag:
+                break
+        }
+    }
+    
+    private func removeInternal(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, range: ClosedRange<MessageId.Id>, operations: inout [MessageHistoryIndexHoleOperationKey: [MessageHistoryIndexHoleOperation]]) {
+        var removeKeys: [Int32] = []
+        var insertRanges = IndexSet()
+        
+        func processRange(_ key: ValueBoxKey, _ value: ReadBuffer) {
+            let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            if range.lowerBound <= holeRange.lowerBound && range.upperBound >= holeRange.upperBound {
+                removeKeys.append(upperId.id)
+            } else if range.overlaps(holeRange) {
+                removeKeys.append(upperId.id)
+                var holeIndices = IndexSet(integersIn: Int(holeRange.lowerBound) ... Int(holeRange.upperBound))
+                holeIndices.remove(integersIn: Int(range.lowerBound) ... Int(range.upperBound))
+                insertRanges.formUnion(holeIndices)
+            }
+        }
+        
+        let lowerScanBound = max(0, range.lowerBound - 2)
+        
+        self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: lowerScanBound), space: space), end: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space).successor, values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 0)
+        
+        self.valueBox.range(self.table, start: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: range.upperBound), space: space), end: self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+            processRange(key, value)
+            return true
+        }, limit: 1)
+        
+        for id in removeKeys {
+            self.valueBox.remove(self.table, key: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: id), space: space), secure: false)
+        }
+        
+        for insertRange in insertRanges.rangeView {
+            let closedRange: ClosedRange<MessageId.Id> = Int32(insertRange.lowerBound) ... Int32(insertRange.upperBound - 1)
+            var lowerBound: Int32 = closedRange.lowerBound
+            self.valueBox.set(self.table, key: self.key(threadId: threadId, id: MessageId(peerId: peerId, namespace: namespace, id: closedRange.upperBound), space: space), value: MemoryBuffer(memory: &lowerBound, capacity: 4, length: 4, freeWhenDone: false))
+        }
+        
+        if !removeKeys.isEmpty {
+            addMessageHistoryHoleOperation(.remove(range), peerId: peerId, threadId: threadId, namespace: namespace, space: MessageHistoryHoleOperationSpace(space), to: &operations)
+        }
+    }
+    
+    func debugList(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace) -> [ClosedRange<MessageId.Id>] {
+        var result: [ClosedRange<MessageId.Id>] = []
+        self.valueBox.range(self.table, start: self.lowerBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), end: self.upperBound(peerId: peerId, threadId: threadId, namespace: namespace, space: space), values: { key, value in
+            let (keyThreadId, upperId, keySpace) = decomposeKey(key)
+            assert(keyThreadId == threadId)
+            assert(keySpace == space)
+            assert(upperId.peerId == peerId)
+            assert(upperId.namespace == namespace)
+            let lowerId = decodeValue(value: value, peerId: peerId, namespace: namespace)
+            let holeRange: ClosedRange<MessageId.Id> = lowerId.id ... upperId.id
+            result.append(holeRange)
+            return true
+        }, limit: 0)
+        return result
+    }
+}

submodules/Postbox/Sources/MessageHistoryThreadIndexView.swift

@@ -0,0 +1,292 @@
+import Foundation
+
+final class MutableMessageHistoryThreadIndexView: MutablePostboxView {
+    final class Item {
+        let id: Int64
+        let pinnedIndex: Int?
+        let index: MessageIndex
+        var info: CodableEntry
+        var tagSummaryInfo: [ChatListEntryMessageTagSummaryKey: ChatListMessageTagSummaryInfo]
+        var topMessage: Message?
+        var embeddedInterfaceState: StoredPeerChatInterfaceState?
+        
+        init(
+            id: Int64,
+            pinnedIndex: Int?,
+            index: MessageIndex,
+            info: CodableEntry,
+            tagSummaryInfo: [ChatListEntryMessageTagSummaryKey: ChatListMessageTagSummaryInfo],
+            topMessage: Message?,
+            embeddedInterfaceState: StoredPeerChatInterfaceState?
+        ) {
+            self.id = id
+            self.pinnedIndex = pinnedIndex
+            self.index = index
+            self.info = info
+            self.tagSummaryInfo = tagSummaryInfo
+            self.topMessage = topMessage
+            self.embeddedInterfaceState = embeddedInterfaceState
+        }
+    }
+    
+    fileprivate let peerId: PeerId
+    fileprivate let summaryComponents: ChatListEntrySummaryComponents
+    fileprivate var peer: Peer?
+    fileprivate var peerNotificationSettings: PeerNotificationSettings?
+    fileprivate var items: [Item] = []
+    private var hole: ForumTopicListHolesEntry?
+    fileprivate var isLoading: Bool = false
+    
+    init(postbox: PostboxImpl, peerId: PeerId, summaryComponents: ChatListEntrySummaryComponents) {
+        self.peerId = peerId
+        self.summaryComponents = summaryComponents
+        
+        self.reload(postbox: postbox)
+    }
+    
+    private func reload(postbox: PostboxImpl) {
+        self.items.removeAll()
+        
+        self.peer = postbox.peerTable.get(self.peerId)
+        
+        self.peerNotificationSettings = postbox.peerNotificationSettingsTable.getEffective(self.peerId)
+        
+        let validIndexBoundary = postbox.peerThreadCombinedStateTable.get(peerId: peerId)?.validIndexBoundary
+        self.isLoading = validIndexBoundary == nil
+        
+        if let validIndexBoundary = validIndexBoundary {
+            if validIndexBoundary.messageId != 1 {
+                self.hole = ForumTopicListHolesEntry(peerId: self.peerId, index: validIndexBoundary)
+            } else {
+                self.hole = nil
+            }
+        } else {
+            self.hole = ForumTopicListHolesEntry(peerId: self.peerId, index: nil)
+        }
+        
+        if !self.isLoading {
+            let pinnedThreadIds = postbox.messageHistoryThreadPinnedTable.get(peerId: self.peerId)
+        
+            for item in postbox.messageHistoryThreadIndexTable.getAll(peerId: self.peerId) {
+                var pinnedIndex: Int?
+                if let index = pinnedThreadIds.firstIndex(of: item.threadId) {
+                    pinnedIndex = index
+                }
+                
+                var tagSummaryInfo: [ChatListEntryMessageTagSummaryKey: ChatListMessageTagSummaryInfo] = [:]
+                for (key, component) in self.summaryComponents.components {
+                    var tagSummaryCount: Int32?
+                    var actionsSummaryCount: Int32?
+                    
+                    if let tagSummary = component.tagSummary {
+                        let key = MessageHistoryTagsSummaryKey(tag: key.tag, peerId: self.peerId, threadId: item.threadId, namespace: tagSummary.namespace, customTag: nil)
+                        if let summary = postbox.messageHistoryTagsSummaryTable.get(key) {
+                            tagSummaryCount = summary.count
+                        }
+                    }
+                    
+                    if let actionsSummary = component.actionsSummary {
+                        let key = PendingMessageActionsSummaryKey(type: key.actionType, peerId: self.peerId, namespace: actionsSummary.namespace)
+                        actionsSummaryCount = postbox.pendingMessageActionsMetadataTable.getCount(.peerNamespaceAction(key.peerId, key.namespace, key.type))
+                    }
+                    
+                    tagSummaryInfo[key] = ChatListMessageTagSummaryInfo(
+                        tagSummaryCount: tagSummaryCount,
+                        actionsSummaryCount: actionsSummaryCount
+                    )
+                }
+                
+                var embeddedInterfaceState: StoredPeerChatInterfaceState?
+                embeddedInterfaceState = postbox.peerChatThreadInterfaceStateTable.get(PeerChatThreadId(peerId: self.peerId, threadId: item.threadId))
+                
+                self.items.append(Item(
+                    id: item.threadId,
+                    pinnedIndex: pinnedIndex,
+                    index: item.index,
+                    info: item.info.data,
+                    tagSummaryInfo: tagSummaryInfo,
+                    topMessage: postbox.getMessage(item.index.id),
+                    embeddedInterfaceState: embeddedInterfaceState
+                ))
+            }
+            
+            self.items.sort(by: { lhs, rhs in
+                if let lhsPinnedIndex = lhs.pinnedIndex, let rhsPinnedIndex = rhs.pinnedIndex {
+                    return lhsPinnedIndex < rhsPinnedIndex
+                } else if (lhs.pinnedIndex == nil) != (rhs.pinnedIndex == nil) {
+                    if lhs.pinnedIndex != nil {
+                        return true
+                    } else {
+                        return false
+                    }
+                }
+                
+                return lhs.index > rhs.index
+            })
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if transaction.updatedMessageThreadPeerIds.contains(self.peerId) || transaction.updatedPinnedThreads.contains(self.peerId) || transaction.updatedPeerThreadCombinedStates.contains(self.peerId) || transaction.currentUpdatedMessageTagSummaries.contains(where: { $0.key.peerId == self.peerId }) || transaction.currentUpdatedMessageActionsSummaries.contains(where: { $0.key.peerId == self.peerId }) || transaction.currentUpdatedPeerChatListEmbeddedStates.contains(self.peerId) || transaction.currentUpdatedPeerNotificationSettings[self.peerId] != nil || transaction.updatedPinnedThreads.contains(self.peerId) {
+            self.reload(postbox: postbox)
+            updated = true
+        }
+        
+        return updated
+    }
+    
+    func topHole() -> ForumTopicListHolesEntry? {
+        return self.hole
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        self.reload(postbox: postbox)
+        
+        return true
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageHistoryThreadIndexView(self)
+    }
+}
+
+public final class EngineMessageHistoryThread {
+    public final class Item: Equatable {
+        public let id: Int64
+        public let pinnedIndex: Int?
+        public let index: MessageIndex
+        public let info: CodableEntry
+        public let tagSummaryInfo: [ChatListEntryMessageTagSummaryKey: ChatListMessageTagSummaryInfo]
+        public let topMessage: Message?
+        public let embeddedInterfaceState: StoredPeerChatInterfaceState?
+        
+        public init(
+            id: Int64,
+            pinnedIndex: Int?,
+            index: MessageIndex,
+            info: CodableEntry,
+            tagSummaryInfo: [ChatListEntryMessageTagSummaryKey: ChatListMessageTagSummaryInfo],
+            topMessage: Message?,
+            embeddedInterfaceState: StoredPeerChatInterfaceState?
+        ) {
+            self.id = id
+            self.pinnedIndex = pinnedIndex
+            self.index = index
+            self.info = info
+            self.tagSummaryInfo = tagSummaryInfo
+            self.topMessage = topMessage
+            self.embeddedInterfaceState = embeddedInterfaceState
+        }
+        
+        public static func ==(lhs: Item, rhs: Item) -> Bool {
+            if lhs.id != rhs.id {
+                return false
+            }
+            if lhs.pinnedIndex != rhs.pinnedIndex {
+                return false
+            }
+            if lhs.index != rhs.index {
+                return false
+            }
+            if lhs.info != rhs.info {
+                return false
+            }
+            if lhs.tagSummaryInfo != rhs.tagSummaryInfo {
+                return false
+            }
+            if let lhsMessage = lhs.topMessage, let rhsMessage = rhs.topMessage {
+                if lhsMessage.index != rhsMessage.index {
+                    return false
+                }
+                if lhsMessage.stableVersion != rhsMessage.stableVersion {
+                    return false
+                }
+            } else if (lhs.topMessage == nil) != (rhs.topMessage == nil) {
+                return false
+            }
+            if lhs.embeddedInterfaceState != rhs.embeddedInterfaceState {
+                return false
+            }
+            
+            return true
+        }
+    }
+}
+
+public final class MessageHistoryThreadIndexView: PostboxView {
+    public let peer: Peer?
+    public let peerNotificationSettings: PeerNotificationSettings?
+    public let items: [EngineMessageHistoryThread.Item]
+    public let isLoading: Bool
+    
+    init(_ view: MutableMessageHistoryThreadIndexView) {
+        self.peer = view.peer
+        self.peerNotificationSettings = view.peerNotificationSettings
+        
+        var items: [EngineMessageHistoryThread.Item] = []
+        for item in view.items {
+            items.append(EngineMessageHistoryThread.Item(
+                id: item.id,
+                pinnedIndex: item.pinnedIndex,
+                index: item.index,
+                info: item.info,
+                tagSummaryInfo: item.tagSummaryInfo,
+                topMessage: item.topMessage,
+                embeddedInterfaceState: item.embeddedInterfaceState
+            ))
+        }
+        self.items = items
+        
+        self.isLoading = view.isLoading
+    }
+}
+
+final class MutableMessageHistoryThreadInfoView: MutablePostboxView {
+    private let peerId: PeerId
+    private let threadId: Int64
+    
+    fileprivate var info: StoredMessageHistoryThreadInfo?
+    
+    init(postbox: PostboxImpl, peerId: PeerId, threadId: Int64) {
+        self.peerId = peerId
+        self.threadId = threadId
+        
+        self.reload(postbox: postbox)
+    }
+    
+    private func reload(postbox: PostboxImpl) {
+        self.info = postbox.messageHistoryThreadIndexTable.get(peerId: self.peerId, threadId: self.threadId)
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var updated = false
+        
+        if transaction.updatedMessageThreadPeerIds.contains(self.peerId) {
+            let info = postbox.messageHistoryThreadIndexTable.get(peerId: self.peerId, threadId: self.threadId)
+            if self.info != info {
+                self.info = info
+                updated = true
+            }
+        }
+        
+        return updated
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageHistoryThreadInfoView(self)
+    }
+}
+
+public final class MessageHistoryThreadInfoView: PostboxView {
+    public let info: StoredMessageHistoryThreadInfo?
+    
+    init(_ view: MutableMessageHistoryThreadInfoView) {
+        self.info = view.info
+    }
+}

submodules/Postbox/Sources/MessageHistoryThreadsTable.swift

@@ -0,0 +1,378 @@
+import Foundation
+
+class MessageHistoryThreadsTable: Table {
+    struct ItemId: Hashable {
+        var peerId: PeerId
+        var threadId: Int64
+    }
+    
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4)
+    
+    private(set) var updatedIds = Set<ItemId>()
+    
+    override init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool) {
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(threadId: Int64, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4)) -> ValueBoxKey {
+        key.setInt64(0, value: index.id.peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: index.id.namespace)
+        key.setInt32(8 + 8 + 4, value: index.timestamp)
+        key.setInt32(8 + 8 + 4 + 4, value: index.id.id)
+        return key
+    }
+    
+    private func extractKey(_ key: ValueBoxKey) -> MessageIndex {
+        return MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 8), id: key.getInt32(8 + 8 + 4 + 4)), timestamp: key.getInt32(8 + 8 + 4))
+    }
+    
+    private func lowerBound(threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setInt32(8 + 8, value: namespace)
+        return key
+    }
+    
+    private func upperBound(threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        return self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace).successor
+    }
+    
+    func add(threadId: Int64, index: MessageIndex) {
+        self.valueBox.set(self.table, key: self.key(threadId: threadId, index: index, key: self.sharedKey), value: MemoryBuffer())
+        self.updatedIds.insert(ItemId(peerId: index.id.peerId, threadId: threadId))
+    }
+    
+    func remove(threadId: Int64, index: MessageIndex) {
+        self.valueBox.remove(self.table, key: self.key(threadId: threadId, index: index, key: self.sharedKey), secure: false)
+        self.updatedIds.insert(ItemId(peerId: index.id.peerId, threadId: threadId))
+    }
+    
+    func earlierIndices(threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, count: Int) -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(threadId: threadId, index: index).successor
+            } else {
+                key = self.key(threadId: threadId, index: index)
+            }
+        } else {
+            key = self.upperBound(threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func laterIndices(threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, count: Int) -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(threadId: threadId, index: index).predecessor
+            } else {
+                key = self.key(threadId: threadId, index: index)
+            }
+        } else {
+            key = self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: self.upperBound(threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func getTop(peerId: PeerId, threadId: Int64, namespaces: Set<MessageId.Namespace>) -> MessageIndex? {
+        var maxIndex: MessageIndex?
+        for namespace in namespaces {
+            var namespaceIndex: MessageIndex?
+            self.valueBox.range(self.table, start: self.upperBound(threadId: threadId, peerId: peerId, namespace: namespace), end: self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+                namespaceIndex = extractKey(key)
+                return false
+            }, limit: 1)
+            if let namespaceIndex = namespaceIndex {
+                if let maxIndexValue = maxIndex {
+                    if namespaceIndex > maxIndexValue {
+                        maxIndex = namespaceIndex
+                    }
+                } else {
+                    maxIndex = namespaceIndex
+                }
+            }
+        }
+        return maxIndex
+    }
+    
+    func holeLowerBoundForTopValidRange(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, space: MessageHistoryHoleSpace, holeIndexTable: MessageHistoryThreadHoleIndexTable) -> MessageId.Id {
+        let topHole = holeIndexTable.latest(peerId: peerId, threadId: threadId, namespace: namespace, space: space)
+        if let topHole = topHole {
+            let maxInHole = topHole.upperBound
+            var messageNotInHole: MessageId?
+            self.valueBox.range(self.table, start: self.upperBound(threadId: threadId, peerId: peerId, namespace: namespace), end: self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+                let index = extractKey(key)
+                if index.id.id > maxInHole {
+                    messageNotInHole = index.id
+                    return false
+                }
+                return true
+            }, limit: 32000)
+            if let messageNotInHole = messageNotInHole {
+                return messageNotInHole.id + 1
+            } else {
+                return topHole.lowerBound
+            }
+        } else {
+            var messageNotInHole: MessageId?
+            self.valueBox.range(self.table, start: self.upperBound(threadId: threadId, peerId: peerId, namespace: namespace), end: self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+                let index = extractKey(key)
+            
+                messageNotInHole = index.id
+                return false
+            }, limit: 1)
+            if let messageNotInHole = messageNotInHole {
+                return messageNotInHole.id + 1
+            } else {
+                return 1
+            }
+        }
+    }
+    
+    func getMessageCountInRange(threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace, lowerBound: MessageIndex?, upperBound: MessageIndex?) -> Int {
+        if let lowerBound = lowerBound {
+            precondition(lowerBound.id.namespace == namespace)
+        }
+        if let upperBound = upperBound {
+            precondition(upperBound.id.namespace == namespace)
+        }
+        var lowerBoundKey: ValueBoxKey
+        if let lowerBound = lowerBound {
+            lowerBoundKey = self.key(threadId: threadId, index: lowerBound)
+            if lowerBound.timestamp > 1 {
+                lowerBoundKey = lowerBoundKey.predecessor
+            }
+        } else {
+            lowerBoundKey = self.lowerBound(threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        var upperBoundKey: ValueBoxKey
+        if let upperBound = upperBound {
+            upperBoundKey = self.key(threadId: threadId, index: upperBound)
+            if upperBound.timestamp < Int32.max - 1 {
+                upperBoundKey = upperBoundKey.successor
+            }
+        } else {
+            upperBoundKey = self.upperBound(threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        return Int(self.valueBox.count(self.table, start: lowerBoundKey, end: upperBoundKey))
+    }
+    
+    override func beforeCommit() {
+        super.beforeCommit()
+        
+        self.updatedIds.removeAll()
+    }
+}
+
+class MessageHistoryThreadTagsTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let summaryTable: MessageHistoryTagsSummaryTable
+    private let summaryTags: MessageTags
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4)
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool, seedConfiguration: SeedConfiguration, summaryTable: MessageHistoryTagsSummaryTable) {
+        self.summaryTable = summaryTable
+        self.summaryTags = seedConfiguration.messageTagsWithSummary
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(tag: MessageTags, threadId: Int64, index: MessageIndex, key: ValueBoxKey = ValueBoxKey(length: 8 + 8 + 4 + 4 + 4 + 4)) -> ValueBoxKey {
+        key.setInt64(0, value: index.id.peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setUInt32(8 + 8, value: tag.rawValue)
+        key.setInt32(8 + 8 + 4, value: index.id.namespace)
+        key.setInt32(8 + 8 + 4 + 4, value: index.timestamp)
+        key.setInt32(8 + 8 + 4 + 4 + 4, value: index.id.id)
+        return key
+    }
+    
+    private func extractKey(_ key: ValueBoxKey) -> MessageIndex {
+        return MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 8 + 4), id: key.getInt32(8 + 8 + 4 + 4 + 4)), timestamp: key.getInt32(8 + 8 + 4 + 4))
+    }
+    
+    private func lowerBound(tag: MessageTags, threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8 + 8 + 4 + 4)
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        key.setUInt32(8 + 8, value: tag.rawValue)
+        key.setInt32(8 + 8 + 4, value: namespace)
+        return key
+    }
+    
+    private func upperBound(tag: MessageTags, threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace) -> ValueBoxKey {
+        return self.lowerBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace).successor
+    }
+    
+    func add(tags: MessageTags, threadId: Int64, index: MessageIndex, isNewlyAdded: Bool, updatedSummaries: inout[MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        for tag in tags {
+            self.valueBox.set(self.table, key: self.key(tag: tag, threadId: threadId, index: index, key: self.sharedKey), value: MemoryBuffer())
+            if self.summaryTags.contains(tag) {
+                self.summaryTable.addMessage(key: MessageHistoryTagsSummaryKey(tag: tag, peerId: index.id.peerId, threadId: threadId, namespace: index.id.namespace, customTag: nil), id: index.id.id, isNewlyAdded: isNewlyAdded, updatedSummaries: &updatedSummaries, invalidateSummaries: &invalidateSummaries)
+            }
+        }
+    }
+    
+    func remove(tags: MessageTags, threadId: Int64, index: MessageIndex, updatedSummaries: inout[MessageHistoryTagsSummaryKey: MessageHistoryTagNamespaceSummary], invalidateSummaries: inout [InvalidatedMessageHistoryTagsSummaryEntryOperation]) {
+        for tag in tags {
+            self.valueBox.remove(self.table, key: self.key(tag: tag, threadId: threadId, index: index, key: self.sharedKey), secure: false)
+            if self.summaryTags.contains(tag) {
+                self.summaryTable.removeMessage(key: MessageHistoryTagsSummaryKey(tag: tag, peerId: index.id.peerId, threadId: threadId, namespace: index.id.namespace, customTag: nil), id: index.id.id, updatedSummaries: &updatedSummaries, invalidateSummaries: &invalidateSummaries)
+            }
+        }
+    }
+    
+    func entryExists(tag: MessageTags, threadId: Int64, index: MessageIndex) -> Bool {
+        return self.valueBox.exists(self.table, key: self.key(tag: tag, threadId: threadId, index: index, key: self.sharedKey))
+    }
+    
+    func entryLocation(at index: MessageIndex, threadId: Int64, tag: MessageTags) -> MessageHistoryEntryLocation? {
+        if let _ = self.valueBox.get(self.table, key: self.key(tag: tag, threadId: threadId, index: index)) {
+            var greaterCount = 0
+            self.valueBox.range(self.table, start: self.key(tag: tag, threadId: threadId, index: index), end: self.upperBound(tag: tag, threadId: threadId, peerId: index.id.peerId, namespace: index.id.namespace), keys: { _ in
+                greaterCount += 1
+                return true
+            }, limit: 0)
+            
+            var lowerCount = 0
+            self.valueBox.range(self.table, start: self.key(tag: tag, threadId: threadId, index: index), end: self.lowerBound(tag: tag, threadId: threadId, peerId: index.id.peerId, namespace: index.id.namespace), keys: { _ in
+                lowerCount += 1
+                return true
+            }, limit: 0)
+            
+            return MessageHistoryEntryLocation(index: lowerCount, count: greaterCount + lowerCount + 1)
+        }
+        return nil
+    }
+    
+    func earlierIndices(tag: MessageTags, threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, minIndex: MessageIndex? = nil, count: Int) -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(tag: tag, threadId: threadId, index: index).successor
+            } else {
+                key = self.key(tag: tag, threadId: threadId, index: index)
+            }
+        } else {
+            key = self.upperBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        let endKey: ValueBoxKey
+        if let minIndex = minIndex {
+            endKey = self.key(tag: tag, threadId: threadId, index: minIndex)
+        } else {
+            endKey = self.lowerBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: endKey, keys: { key in
+            indices.append(self.extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func laterIndices(tag: MessageTags, threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace, index: MessageIndex?, includeFrom: Bool, count: Int) -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        let key: ValueBoxKey
+        if let index = index {
+            if includeFrom {
+                key = self.key(tag: tag, threadId: threadId, index: index).predecessor
+            } else {
+                key = self.key(tag: tag, threadId: threadId, index: index)
+            }
+        } else {
+            key = self.lowerBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace)
+        }
+        self.valueBox.range(self.table, start: key, end: self.upperBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(self.extractKey(key))
+            return true
+        }, limit: count)
+        return indices
+    }
+    
+    func getMessageCountInRange(tag: MessageTags, threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace, lowerBound: MessageIndex, upperBound: MessageIndex) -> Int {
+        precondition(lowerBound.id.namespace == namespace)
+        precondition(upperBound.id.namespace == namespace)
+        var lowerBoundKey = self.key(tag: tag, threadId: threadId, index: lowerBound)
+        if lowerBound.timestamp > 1 {
+            lowerBoundKey = lowerBoundKey.predecessor
+        }
+        var upperBoundKey = self.key(tag: tag, threadId: threadId, index: upperBound)
+        if upperBound.timestamp < Int32.max - 1 {
+            upperBoundKey = upperBoundKey.successor
+        }
+        return Int(self.valueBox.count(self.table, start: lowerBoundKey, end: upperBoundKey))
+    }
+    
+    func latestIndex(tag: MessageTags, threadId: Int64, peerId: PeerId, namespace: MessageId.Namespace) -> MessageIndex? {
+        var result: MessageIndex?
+        self.valueBox.range(self.table, start: self.lowerBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace), end: self.upperBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+            result = extractKey(key)
+            return true
+        }, limit: 1)
+        return result
+    }
+    
+    func findRandomIndex(peerId: PeerId, threadId: Int64, namespace: MessageId.Namespace, tag: MessageTags, ignoreIds: ([MessageId], Set<MessageId>), isMessage: (MessageIndex) -> Bool) -> MessageIndex? {
+        var indices: [MessageIndex] = []
+        self.valueBox.range(self.table, start: self.lowerBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace), end: self.upperBound(tag: tag, threadId: threadId, peerId: peerId, namespace: namespace), keys: { key in
+            indices.append(extractKey(key))
+            return true
+        }, limit: 0)
+        var checkedIndices = Set<Int>()
+        while checkedIndices.count < indices.count {
+            let i = Int(arc4random_uniform(UInt32(indices.count)))
+            if checkedIndices.contains(i) {
+                continue
+            }
+            checkedIndices.insert(i)
+            let index = indices[i]
+            if isMessage(index) && !ignoreIds.1.contains(index.id) {
+                return index
+            }
+        }
+        checkedIndices.removeAll()
+        let lastId = ignoreIds.0.last
+        while checkedIndices.count < indices.count {
+            let i = Int(arc4random_uniform(UInt32(indices.count)))
+            if checkedIndices.contains(i) {
+                continue
+            }
+            checkedIndices.insert(i)
+            let index = indices[i]
+            if isMessage(index) && lastId != index.id {
+                return index
+            }
+        }
+        return nil
+    }
+    
+    func debugGetAllIndices() -> [MessageIndex] {
+        var indices: [MessageIndex] = []
+        self.valueBox.scan(self.table, values: { key, value in
+            indices.append(extractKey(key))
+            return true
+        })
+        return indices
+    }
+}

submodules/Postbox/Sources/MessageHistoryUnsentTable.swift

@@ -0,0 +1,57 @@
+import Foundation
+
+enum IntermediateMessageHistoryUnsentOperation {
+    case Insert(MessageId)
+    case Remove(MessageId)
+}
+
+final class MessageHistoryUnsentTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 4 + 4 + 8)
+    
+    private func key(_ id: MessageId) -> ValueBoxKey {
+        self.sharedKey.setInt32(0, value: id.namespace)
+        self.sharedKey.setInt32(4, value: id.id)
+        self.sharedKey.setInt64(4 + 4, value: id.peerId.toInt64())
+        
+        return self.sharedKey
+    }
+    
+    private func lowerBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 1)
+        key.setInt8(0, value: 0)
+        return key
+    }
+    
+    private func upperBound() -> ValueBoxKey {
+        let key = ValueBoxKey(length: 4 + 4 + 8)
+        memset(key.memory, 0xff, key.length)
+        return key
+    }
+    
+    func add(_ id: MessageId, operations: inout [IntermediateMessageHistoryUnsentOperation]) {
+        self.valueBox.set(self.table, key: self.key(id), value: MemoryBuffer())
+        operations.append(.Insert(id))
+    }
+    
+    func remove(_ id: MessageId, operations: inout [IntermediateMessageHistoryUnsentOperation]) {
+        self.valueBox.remove(self.table, key: self.key(id), secure: false)
+        operations.append(.Remove(id))
+    }
+    
+    func get() -> [MessageId] {
+        var ids: [MessageId] = []
+        self.valueBox.range(self.table, start: self.lowerBound(), end: self.upperBound(), keys: { key in
+            ids.append(MessageId(peerId: PeerId(key.getInt64(4 + 4)), namespace: key.getInt32(0), id: key.getInt32(4)))
+            return true
+        }, limit: 0)
+        return ids
+    }
+    
+    override func beforeCommit() {
+        
+    }
+}

submodules/Postbox/Sources/MessageHistoryViewEntryAttributes.swift

@@ -0,0 +1,9 @@
+import Foundation
+
+public struct MutableMessageHistoryEntryAttributes: Equatable {
+    public var authorIsContact: Bool
+    
+    public init(authorIsContact: Bool) {
+        self.authorIsContact = authorIsContact
+    }
+}

submodules/Postbox/Sources/MessageMediaTable.swift

@@ -0,0 +1,321 @@
+import Foundation
+
+private enum MediaEntryType: Int8 {
+    case Direct
+    case MessageReference
+}
+
+enum InsertMediaResult {
+    case Reference
+    case Embed(Media)
+}
+
+enum RemoveMediaResult {
+    case Reference
+    case Embedded(MessageIndex)
+}
+
+enum DebugMediaEntry {
+    case Direct(Media, Int)
+    case MessageReference(MessageIndex)
+}
+
+final class MessageMediaTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: false)
+    }
+
+    func key(_ id: MediaId, key: ValueBoxKey = ValueBoxKey(length: 4 + 8)) -> ValueBoxKey {
+        key.setInt32(0, value: id.namespace)
+        key.setInt64(4, value: id.id)
+        return key
+    }
+    
+    func exists(id: MediaId) -> Bool {
+        return self.valueBox.exists(self.table, key: self.key(id))
+    }
+    
+    func get(_ id: MediaId, embedded: (MessageIndex, MediaId) -> Media?) -> (MessageIndex?, Media)? {
+        if let value = self.valueBox.get(self.table, key: self.key(id)) {
+            var type: Int8 = 0
+            value.read(&type, offset: 0, length: 1)
+            if type == MediaEntryType.Direct.rawValue {
+                var dataLength: Int32 = 0
+                value.read(&dataLength, offset: 0, length: 4)
+                if let media = PostboxDecoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                    return (nil, media)
+                }
+            } else if type == MediaEntryType.MessageReference.rawValue {
+                var idPeerId: Int64 = 0
+                var idNamespace: Int32 = 0
+                var idId: Int32 = 0
+                var idTimestamp: Int32 = 0
+                value.read(&idPeerId, offset: 0, length: 8)
+                value.read(&idNamespace, offset: 0, length: 4)
+                value.read(&idId, offset: 0, length: 4)
+                value.read(&idTimestamp, offset: 0, length: 4)
+                
+                let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                
+                if let result = embedded(referencedMessageIndex, id) {
+                    return (referencedMessageIndex, result)
+                } else {
+                    return nil
+                }
+            }
+        }
+        return nil
+    }
+    
+    func set(_ media: Media, index: MessageIndex?, messageHistoryTable: MessageHistoryTable, sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: PostboxEncoder = PostboxEncoder()) -> InsertMediaResult {
+        if let id = media.id {
+            if let value = self.valueBox.get(self.table, key: self.key(id)) {
+                var type: Int8 = 0
+                value.read(&type, offset: 0, length: 1)
+                if type == MediaEntryType.Direct.rawValue {
+                    var dataLength: Int32 = 0
+                    value.read(&dataLength, offset: 0, length: 4)
+                    value.skip(Int(dataLength))
+                    
+                    sharedWriteBuffer.reset()
+                    sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset)
+                    
+                    var messageReferenceCount: Int32 = 0
+                    value.read(&messageReferenceCount, offset: 0, length: 4)
+                    messageReferenceCount += 1
+                    sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    withExtendedLifetime(sharedWriteBuffer, {
+                        self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                    })
+                    
+                    return .Reference
+                } else if type == MediaEntryType.MessageReference.rawValue {
+                    var idPeerId: Int64 = 0
+                    var idNamespace: Int32 = 0
+                    var idId: Int32 = 0
+                    var idTimestamp: Int32 = 0
+                    value.read(&idPeerId, offset: 0, length: 8)
+                    value.read(&idNamespace, offset: 0, length: 4)
+                    value.read(&idId, offset: 0, length: 4)
+                    value.read(&idTimestamp, offset: 0, length: 4)
+                    
+                    let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                    if referencedMessageIndex == index {
+                        return .Embed(media)
+                    }
+                    
+                    if let media = messageHistoryTable.unembedMedia(referencedMessageIndex, id: id) {
+                        sharedWriteBuffer.reset()
+                        var directType: Int8 = MediaEntryType.Direct.rawValue
+                        sharedWriteBuffer.write(&directType, offset: 0, length: 1)
+                        
+                        sharedEncoder.reset()
+                        sharedEncoder.encodeRootObject(media)
+                        let mediaBuffer = sharedEncoder.memoryBuffer()
+                        var mediaBufferLength = Int32(mediaBuffer.length)
+                        sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4)
+                        sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length)
+                        
+                        var messageReferenceCount: Int32 = 2
+                        sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                        
+                        withExtendedLifetime(sharedWriteBuffer, {
+                            self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                        })
+                    }
+                    
+                    return .Reference
+                } else {
+                    return .Embed(media)
+                }
+            } else {
+                if let index = index {
+                    sharedWriteBuffer.reset()
+                    var type: Int8 = MediaEntryType.MessageReference.rawValue
+                    sharedWriteBuffer.write(&type, offset: 0, length: 1)
+                    var idPeerId: Int64 = index.id.peerId.toInt64()
+                    var idNamespace: Int32 = index.id.namespace
+                    var idId: Int32 = index.id.id
+                    var idTimestamp: Int32 = index.timestamp
+                    sharedWriteBuffer.write(&idPeerId, offset: 0, length: 8)
+                    sharedWriteBuffer.write(&idNamespace, offset: 0, length: 4)
+                    sharedWriteBuffer.write(&idId, offset: 0, length: 4)
+                    sharedWriteBuffer.write(&idTimestamp, offset: 0, length: 4)
+                    
+                    withExtendedLifetime(sharedWriteBuffer, {
+                        self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                    })
+                    
+                    return .Embed(media)
+                } else {
+                    sharedWriteBuffer.reset()
+                    var directType: Int8 = MediaEntryType.Direct.rawValue
+                    sharedWriteBuffer.write(&directType, offset: 0, length: 1)
+                    
+                    sharedEncoder.reset()
+                    sharedEncoder.encodeRootObject(media)
+                    let mediaBuffer = sharedEncoder.memoryBuffer()
+                    var mediaBufferLength = Int32(mediaBuffer.length)
+                    sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4)
+                    sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length)
+                    
+                    var messageReferenceCount: Int32 = 2
+                    sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    withExtendedLifetime(sharedWriteBuffer, {
+                        self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                    })
+                    
+                    return .Reference
+                }
+            }
+        } else {
+            return .Embed(media)
+        }
+    }
+    
+    func removeReference(_ id: MediaId, sharedWriteBuffer: WriteBuffer = WriteBuffer()) -> RemoveMediaResult {
+        if let value = self.valueBox.get(self.table, key: self.key(id)) {
+            var type: Int8 = 0
+            value.read(&type, offset: 0, length: 1)
+            if type == MediaEntryType.Direct.rawValue {
+                var dataLength: Int32 = 0
+                value.read(&dataLength, offset: 0, length: 4)
+                value.skip(Int(dataLength))
+                
+                sharedWriteBuffer.reset()
+                sharedWriteBuffer.write(value.memory, offset: 0, length: value.offset)
+                
+                var messageReferenceCount: Int32 = 0
+                value.read(&messageReferenceCount, offset: 0, length: 4)
+                messageReferenceCount -= 1
+                sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                
+                if messageReferenceCount <= 0 {
+                    self.valueBox.remove(self.table, key: self.key(id), secure: false)
+                } else {
+                    withExtendedLifetime(sharedWriteBuffer, {
+                        self.valueBox.set(self.table, key: self.key(id), value: sharedWriteBuffer.readBufferNoCopy())
+                    })
+                }
+                
+                return .Reference
+            } else if type == MediaEntryType.MessageReference.rawValue {
+                var idPeerId: Int64 = 0
+                var idNamespace: Int32 = 0
+                var idId: Int32 = 0
+                var idTimestamp: Int32 = 0
+                value.read(&idPeerId, offset: 0, length: 8)
+                value.read(&idNamespace, offset: 0, length: 4)
+                value.read(&idId, offset: 0, length: 4)
+                value.read(&idTimestamp, offset: 0, length: 4)
+                
+                let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                
+                self.valueBox.remove(self.table, key: self.key(id), secure: false)
+                
+                return .Embedded(referencedMessageIndex)
+            } else {
+                assertionFailure()
+            }
+        }
+        return .Reference
+    }
+    
+    func removeEmbeddedMedia(_ media: Media) {
+        if let id = media.id {
+            self.valueBox.remove(self.table, key: self.key(id), secure: false)
+        }
+    }
+    
+    func update(_ id: MediaId, media: Media, messageHistoryTable: MessageHistoryTable, operationsByPeerId: inout [PeerId: [MessageHistoryOperation]], sharedWriteBuffer: WriteBuffer = WriteBuffer(), sharedEncoder: PostboxEncoder = PostboxEncoder())  {
+        if let updatedId = media.id {
+            if let value = self.valueBox.get(self.table, key: self.key(id)) {
+                var type: Int8 = 0
+                value.read(&type, offset: 0, length: 1)
+                if type == MediaEntryType.Direct.rawValue {
+                    var dataLength: Int32 = 0
+                    value.read(&dataLength, offset: 0, length: 4)
+                    value.skip(Int(dataLength))
+                    
+                    var messageReferenceCount: Int32 = 0
+                    value.read(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    sharedWriteBuffer.reset()
+                    var directType: Int8 = MediaEntryType.Direct.rawValue
+                    sharedWriteBuffer.write(&directType, offset: 0, length: 1)
+                    
+                    sharedEncoder.reset()
+                    sharedEncoder.encodeRootObject(media)
+                    let mediaBuffer = sharedEncoder.memoryBuffer()
+                    var mediaBufferLength = Int32(mediaBuffer.length)
+                    sharedWriteBuffer.write(&mediaBufferLength, offset: 0, length: 4)
+                    sharedWriteBuffer.write(mediaBuffer.memory, offset: 0, length: mediaBuffer.length)
+                    
+                    sharedWriteBuffer.write(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    if id != updatedId {
+                        self.valueBox.remove(self.table, key: self.key(id), secure: false)
+                    }
+                    withExtendedLifetime(sharedWriteBuffer, {
+                        self.valueBox.set(self.table, key: self.key(updatedId), value: sharedWriteBuffer.readBufferNoCopy())
+                    })
+                } else if type == MediaEntryType.MessageReference.rawValue {
+                    var idPeerId: Int64 = 0
+                    var idNamespace: Int32 = 0
+                    var idId: Int32 = 0
+                    var idTimestamp: Int32 = 0
+                    value.read(&idPeerId, offset: 0, length: 8)
+                    value.read(&idNamespace, offset: 0, length: 4)
+                    value.read(&idId, offset: 0, length: 4)
+                    value.read(&idTimestamp, offset: 0, length: 4)
+                    
+                    let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                    messageHistoryTable.updateEmbeddedMedia(referencedMessageIndex, mediaId: id, media: media, operationsByPeerId: &operationsByPeerId)
+                }
+            }
+        }
+    }
+    
+    func debugList() -> [DebugMediaEntry] {
+        var entries: [DebugMediaEntry] = []
+        
+        let upperBoundKey = ValueBoxKey(length: 8 + 4)
+        memset(upperBoundKey.memory, 0xff, 8 + 4)
+        self.valueBox.range(self.table, start: ValueBoxKey(length: 0), end: upperBoundKey, values: { key, value in
+            var type: Int8 = 0
+            value.read(&type, offset: 0, length: 1)
+            if type == MediaEntryType.Direct.rawValue {
+                var dataLength: Int32 = 0
+                value.read(&dataLength, offset: 0, length: 4)
+                if let media = PostboxDecoder(buffer: MemoryBuffer(memory: value.memory + value.offset, capacity: Int(dataLength), length: Int(dataLength), freeWhenDone: false)).decodeRootObject() as? Media {
+                    
+                    value.skip(Int(dataLength))
+                    
+                    var messageReferenceCount: Int32 = 0
+                    value.read(&messageReferenceCount, offset: 0, length: 4)
+                    
+                    entries.append(.Direct(media, Int(messageReferenceCount)))
+                }
+            } else if type == MediaEntryType.MessageReference.rawValue {
+                var idPeerId: Int64 = 0
+                var idNamespace: Int32 = 0
+                var idId: Int32 = 0
+                var idTimestamp: Int32 = 0
+                value.read(&idPeerId, offset: 0, length: 8)
+                value.read(&idNamespace, offset: 0, length: 4)
+                value.read(&idId, offset: 0, length: 4)
+                value.read(&idTimestamp, offset: 0, length: 4)
+                
+                let referencedMessageIndex = MessageIndex(id: MessageId(peerId: PeerId(idPeerId), namespace: idNamespace, id: idId), timestamp: idTimestamp)
+                
+                entries.append(.MessageReference(referencedMessageIndex))
+            }
+            
+            return true
+        }, limit: 1000)
+        
+        return entries
+    }
+}

submodules/Postbox/Sources/MessageOfInterestHolesView.swift

@@ -0,0 +1,196 @@
+import Foundation
+
+public struct HolesViewMedia: Comparable {
+    public let media: Media
+    public let peer: Peer
+    public let authorIsContact: Bool
+    public let index: MessageIndex
+    
+    public static func ==(lhs: HolesViewMedia, rhs: HolesViewMedia) -> Bool {
+        return lhs.index == rhs.index && (lhs.media === rhs.media || lhs.media.isEqual(to: rhs.media)) && lhs.peer.isEqual(rhs.peer) && lhs.authorIsContact == rhs.authorIsContact
+    }
+    
+    public static func <(lhs: HolesViewMedia, rhs: HolesViewMedia) -> Bool {
+        return lhs.index < rhs.index
+    }
+}
+
+public struct MessageOfInterestHole: Hashable, Equatable, CustomStringConvertible {
+    public let hole: MessageHistoryViewHole
+    public let direction: MessageHistoryViewRelativeHoleDirection
+    
+    public var description: String {
+        return "hole: \(self.hole), direction: \(self.direction)"
+    }
+}
+
+public enum MessageOfInterestViewLocation: Hashable {
+    case peer(peerId: PeerId, threadId: Int64?)
+}
+
+final class MutableMessageOfInterestHolesView: MutablePostboxView {
+    private let location: MessageOfInterestViewLocation
+    private let count: Int
+    private var anchor: HistoryViewInputAnchor
+    private var wrappedView: MutableMessageHistoryView
+    private var peerIds: MessageHistoryViewInput
+    
+    fileprivate var closestHole: MessageOfInterestHole?
+    fileprivate var closestLaterMedia: [HolesViewMedia] = []
+    
+    init(postbox: PostboxImpl, location: MessageOfInterestViewLocation, namespace: MessageId.Namespace, count: Int) {
+        self.location = location
+        self.count = count
+        
+        let mainPeerId: PeerId
+        let peerIds: MessageHistoryViewInput
+        switch self.location {
+        case let .peer(id, threadId):
+            mainPeerId = id
+            peerIds = postbox.peerIdsForLocation(.peer(peerId: id, threadId: threadId), ignoreRelatedChats: false)
+        }
+        self.peerIds = peerIds
+        var anchor: HistoryViewInputAnchor = .upperBound
+        if let combinedState = postbox.readStateTable.getCombinedState(mainPeerId), let state = combinedState.states.first, state.1.count != 0 {
+            switch state.1 {
+            case let .idBased(maxIncomingReadId, _, _, _, _):
+                anchor = .message(MessageId(peerId: mainPeerId, namespace: state.0, id: maxIncomingReadId))
+            case let .indexBased(maxIncomingReadIndex, _, _, _):
+                anchor = .index(maxIncomingReadIndex)
+            }
+        }
+        self.anchor = anchor
+        self.wrappedView = MutableMessageHistoryView(postbox: postbox, orderStatistics: [], clipHoles: true, trackHoles: true, peerIds: peerIds, ignoreMessagesInTimestampRange: nil, ignoreMessageIds: Set(), anchor: self.anchor, combinedReadStates: nil, transientReadStates: nil, tag: nil, appendMessagesFromTheSameGroup: false, namespaces: .all, count: self.count, topTaggedMessages: [:], additionalDatas: [])
+        let _ = self.updateFromView()
+    }
+    
+    private func updateFromView() -> Bool {
+        let closestHole: MessageOfInterestHole?
+        if let (hole, direction, _, _) = self.wrappedView.firstHole() {
+            closestHole = MessageOfInterestHole(hole: hole, direction: direction)
+        } else {
+            closestHole = nil
+        }
+        
+        var closestLaterMedia: [HolesViewMedia] = []
+        switch self.wrappedView.sampledState {
+        case .loading:
+            break
+        case let .loaded(sample):
+            switch sample.anchor {
+            case .index:
+                let anchorIndex = binaryIndexOrLower(sample.entries, sample.anchor)
+                loop: for i in max(0, anchorIndex) ..< sample.entries.count {
+                    let message = sample.entries[i].message
+                    if !message.media.isEmpty, let peer = message.peers[message.id.peerId] {
+                        for media in message.media {
+                            closestLaterMedia.append(HolesViewMedia(media: media, peer: peer, authorIsContact: sample.entries[i].attributes.authorIsContact, index: message.index))
+                        }
+                    }
+                    if closestLaterMedia.count >= 3 {
+                        break loop
+                    }
+                }
+            case .lowerBound, .upperBound:
+                break
+            }
+        }
+        
+        if self.closestHole != closestHole || self.closestLaterMedia != closestLaterMedia {
+            self.closestHole = closestHole
+            self.closestLaterMedia = closestLaterMedia
+            return true
+        } else {
+            return false
+        }
+    }
+    
+    func replay(postbox: PostboxImpl, transaction: PostboxTransaction) -> Bool {
+        var peerId: PeerId
+        var threadId: Int64?
+        switch self.location {
+        case let .peer(id, threadIdValue):
+            peerId = id
+            threadId = threadIdValue
+        }
+        var anchor: HistoryViewInputAnchor = self.anchor
+        if threadId == nil, transaction.alteredInitialPeerCombinedReadStates[peerId] != nil {
+            let updatedAnchor: HistoryViewInputAnchor = .upperBound
+            if let combinedState = postbox.readStateTable.getCombinedState(peerId), let state = combinedState.states.first, state.1.count != 0 {
+                switch state.1 {
+                case let .idBased(maxIncomingReadId, _, _, _, _):
+                    anchor = .message(MessageId(peerId: peerId, namespace: state.0, id: maxIncomingReadId))
+                case let .indexBased(maxIncomingReadIndex, _, _, _):
+                    anchor = .index(maxIncomingReadIndex)
+                }
+            }
+            anchor = updatedAnchor
+        }
+        
+        if self.anchor != anchor {
+            self.anchor = anchor
+            let peerIds: MessageHistoryViewInput
+            switch self.location {
+            case let .peer(id, threadId):
+                peerIds = postbox.peerIdsForLocation(.peer(peerId: id, threadId: threadId), ignoreRelatedChats: false)
+            }
+            self.wrappedView = MutableMessageHistoryView(postbox: postbox, orderStatistics: [], clipHoles: true, trackHoles: false, peerIds: peerIds, ignoreMessagesInTimestampRange: nil, ignoreMessageIds: Set(), anchor: self.anchor, combinedReadStates: nil, transientReadStates: nil, tag: nil, appendMessagesFromTheSameGroup: false, namespaces: .all, count: self.count, topTaggedMessages: [:], additionalDatas: [])
+            return self.updateFromView()
+        } else if self.wrappedView.replay(postbox: postbox, transaction: transaction) {
+            var reloadView = false
+            if !transaction.currentPeerHoleOperations.isEmpty {
+                var allPeerIds: [PeerId]
+                var threadId: Int64?
+                switch peerIds {
+                case let .single(peerId, threadIdValue):
+                    allPeerIds = [peerId]
+                    threadId = threadIdValue
+                case let .associated(peerId, attachedMessageId):
+                    allPeerIds = [peerId]
+                    if let attachedMessageId = attachedMessageId {
+                        allPeerIds.append(attachedMessageId.peerId)
+                    }
+                case .external:
+                    allPeerIds = []
+                    break
+                }
+                for (key, _) in transaction.currentPeerHoleOperations {
+                    if allPeerIds.contains(key.peerId) && key.threadId == threadId {
+                        reloadView = true
+                        break
+                    }
+                }
+            }
+            if reloadView {
+                let peerIds: MessageHistoryViewInput
+                switch self.location {
+                case let .peer(id, threadId):
+                    peerIds = postbox.peerIdsForLocation(.peer(peerId: id, threadId: threadId), ignoreRelatedChats: false)
+                }
+                self.wrappedView = MutableMessageHistoryView(postbox: postbox, orderStatistics: [], clipHoles: true, trackHoles: false, peerIds: peerIds, ignoreMessagesInTimestampRange: nil, ignoreMessageIds: Set(), anchor: self.anchor, combinedReadStates: nil, transientReadStates: nil, tag: nil, appendMessagesFromTheSameGroup: false, namespaces: .all, count: self.count, topTaggedMessages: [:], additionalDatas: [])
+            }
+            
+            return self.updateFromView()
+        } else {
+            return false
+        }
+    }
+
+    func refreshDueToExternalTransaction(postbox: PostboxImpl) -> Bool {
+        return false
+    }
+    
+    func immutableView() -> PostboxView {
+        return MessageOfInterestHolesView(self)
+    }
+}
+
+public final class MessageOfInterestHolesView: PostboxView {
+    public let closestHole: MessageOfInterestHole?
+    public let closestLaterMedia: [HolesViewMedia]
+    
+    init(_ view: MutableMessageOfInterestHolesView) {
+        self.closestHole = view.closestHole
+        self.closestLaterMedia = view.closestLaterMedia
+    }
+}

submodules/Postbox/Sources/MessageThreadIndexTable.swift

@@ -0,0 +1,378 @@
+import Foundation
+
+public struct StoredMessageHistoryThreadInfo: Equatable, PostboxCoding {
+    public struct Summary: Equatable, PostboxCoding {
+        public var totalUnreadCount: Int32
+        public var isMarkedUnread: Bool
+        public var mutedUntil: Int32?
+        public var maxOutgoingReadId: Int32
+        
+        public init(totalUnreadCount: Int32, isMarkedUnread: Bool, mutedUntil: Int32?, maxOutgoingReadId: Int32) {
+            self.totalUnreadCount = totalUnreadCount
+            self.isMarkedUnread = isMarkedUnread
+            self.mutedUntil = mutedUntil
+            self.maxOutgoingReadId = maxOutgoingReadId
+        }
+        
+        public init(decoder: PostboxDecoder) {
+            self.totalUnreadCount = decoder.decodeInt32ForKey("u", orElse: 0)
+            self.mutedUntil = decoder.decodeOptionalInt32ForKey("m")
+            self.isMarkedUnread = decoder.decodeBoolForKey("mu", orElse: false)
+            self.maxOutgoingReadId = decoder.decodeInt32ForKey("or", orElse: 0)
+        }
+        
+        public func encode(_ encoder: PostboxEncoder) {
+            encoder.encodeInt32(self.totalUnreadCount, forKey: "u")
+            encoder.encodeBool(self.isMarkedUnread, forKey: "mu")
+            if let mutedUntil = self.mutedUntil {
+                encoder.encodeInt32(mutedUntil, forKey: "m")
+            } else {
+                encoder.encodeNil(forKey: "m")
+            }
+            encoder.encodeInt32(self.maxOutgoingReadId, forKey: "or")
+        }
+    }
+    
+    public var data: CodableEntry
+    public var summary: Summary
+    
+    public init(data: CodableEntry, summary: Summary) {
+        self.data = data
+        self.summary = summary
+    }
+    
+    public init(decoder: PostboxDecoder) {
+        self.data = CodableEntry(data: decoder.decodeDataForKey("d")!)
+        self.summary = decoder.decodeObjectForKey("s", decoder: { return Summary(decoder: $0) }) as! Summary
+    }
+    
+    public func encode(_ encoder: PostboxEncoder) {
+        encoder.encodeData(self.data.data, forKey: "d")
+        encoder.encodeObject(self.summary, forKey: "s")
+    }
+}
+
+private func extractKey(_ key: ValueBoxKey) -> MessageIndex {
+    return MessageIndex(id: MessageId(peerId: PeerId(key.getInt64(0)), namespace: key.getInt32(8 + 8), id: key.getInt32(8 + 8 + 4 + 4)), timestamp: key.getInt32(8 + 8 + 4))
+}
+
+class MessageHistoryThreadReverseIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 8)
+    
+    override init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool) {
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(peerId: PeerId, threadId: Int64, key: ValueBoxKey) -> ValueBoxKey {
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt64(8, value: threadId)
+        
+        return key
+    }
+    
+    func get(peerId: PeerId, threadId: Int64) -> MessageIndex? {
+        if let value = self.valueBox.get(self.table, key: self.key(peerId: peerId, threadId: threadId, key: self.sharedKey)) {
+            var result: MessageIndex?
+            withExtendedLifetime(value, {
+                let readBuffer = ReadBuffer(memoryBufferNoCopy: value)
+                var namespace: Int32 = 0
+                readBuffer.read(&namespace, offset: 0, length: 4)
+                var id: Int32 = 0
+                readBuffer.read(&id, offset: 0, length: 4)
+                var timestamp: Int32 = 0
+                readBuffer.read(&timestamp, offset: 0, length: 4)
+                result = MessageIndex(id: MessageId(peerId: peerId, namespace: namespace, id: id), timestamp: timestamp)
+            })
+            return result
+        } else {
+            return nil
+        }
+    }
+    
+    func set(peerId: PeerId, threadId: Int64, timestamp: Int32, namespace: MessageId.Namespace, id: MessageId.Id, hasValue: Bool) {
+        if hasValue {
+            let buffer = WriteBuffer()
+            var namespace = namespace
+            buffer.write(&namespace, length: 4)
+            var id = id
+            buffer.write(&id, length: 4)
+            var timestamp = timestamp
+            buffer.write(&timestamp, length: 4)
+            withExtendedLifetime(buffer, {
+                self.valueBox.set(self.table, key: self.key(peerId: peerId, threadId: threadId, key: self.sharedKey), value: buffer.readBufferNoCopy())
+            })
+        } else {
+            self.valueBox.remove(self.table, key: self.key(peerId: peerId, threadId: threadId, key: self.sharedKey), secure: false)
+        }
+    }
+}
+
+class MessageHistoryThreadIndexTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    struct UpdatedEntry {
+        var value: StoredMessageHistoryThreadInfo?
+    }
+    
+    enum IndexBoundary {
+        case lowerBound
+        case upperBound
+        case index(StoredPeerThreadCombinedState.Index)
+    }
+    
+    private let reverseIndexTable: MessageHistoryThreadReverseIndexTable
+    private let seedConfiguration: SeedConfiguration
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 4 + 8 + 4 + 4)
+    
+    private(set) var updatedInfoItems: [MessageHistoryThreadsTable.ItemId: UpdatedEntry] = [:]
+    
+    init(valueBox: ValueBox, table: ValueBoxTable, reverseIndexTable: MessageHistoryThreadReverseIndexTable, seedConfiguration: SeedConfiguration, useCaches: Bool) {
+        self.reverseIndexTable = reverseIndexTable
+        self.seedConfiguration = seedConfiguration
+        
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(peerId: PeerId, timestamp: Int32, threadId: Int64, namespace: MessageId.Namespace, id: MessageId.Id, key: ValueBoxKey) -> ValueBoxKey {
+        key.setInt64(0, value: peerId.toInt64())
+        key.setInt32(8, value: timestamp)
+        key.setInt64(8 + 4, value: threadId)
+        key.setInt32(8 + 4 + 8, value: namespace)
+        key.setInt32(8 + 4 + 8 + 4, value: id)
+        
+        return key
+    }
+    
+    private static func extract(key: ValueBoxKey) -> (threadId: Int64, index: MessageIndex) {
+        return (
+            threadId: key.getInt64(8 + 4),
+            index: MessageIndex(
+                id: MessageId(
+                    peerId: PeerId(key.getInt64(0)),
+                    namespace: key.getInt32(8 + 4 + 8),
+                    id: key.getInt32(8 + 4 + 8 + 4)
+                ),
+                timestamp: key.getInt32(8)
+            )
+        )
+    }
+    
+    private func lowerBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId).successor
+    }
+    
+    func get(peerId: PeerId, threadId: Int64) -> StoredMessageHistoryThreadInfo? {
+        if let updated = self.updatedInfoItems[MessageHistoryThreadsTable.ItemId(peerId: peerId, threadId: threadId)] {
+            return updated.value
+        } else {
+            if let itemIndex = self.reverseIndexTable.get(peerId: peerId, threadId: threadId) {
+                if let value = self.valueBox.get(self.table, key: self.key(peerId: itemIndex.id.peerId, timestamp: itemIndex.timestamp, threadId: threadId, namespace: itemIndex.id.namespace, id: itemIndex.id.id, key: self.sharedKey)) {
+                    if value.length != 0 {
+                        let decoder = PostboxDecoder(buffer: value)
+                        let state = StoredMessageHistoryThreadInfo(decoder: decoder)
+                        return state
+                    } else {
+                        return nil
+                    }
+                } else {
+                    return nil
+                }
+            } else {
+                return nil
+            }
+        }
+    }
+    
+    func set(peerId: PeerId, threadId: Int64, info: StoredMessageHistoryThreadInfo?) {
+        self.updatedInfoItems[MessageHistoryThreadsTable.ItemId(peerId: peerId, threadId: threadId)] = UpdatedEntry(value: info)
+    }
+    
+    func replay(threadsTable: MessageHistoryThreadsTable, namespaces: Set<MessageId.Namespace>, updatedIds: Set<MessageHistoryThreadsTable.ItemId>) -> Set<PeerId> {
+        var peerIds = Set<PeerId>()
+        for itemId in updatedIds.union(Set(self.updatedInfoItems.keys)) {
+            let topIndex = threadsTable.getTop(peerId: itemId.peerId, threadId: itemId.threadId, namespaces: namespaces)
+            let previousIndex = self.reverseIndexTable.get(peerId: itemId.peerId, threadId: itemId.threadId)
+            if topIndex != previousIndex || self.updatedInfoItems[itemId] != nil {
+                peerIds.insert(itemId.peerId)
+                
+                var info: ReadBuffer?
+                if let previousIndex = previousIndex {
+                    let previousKey = self.key(peerId: itemId.peerId, timestamp: previousIndex.timestamp, threadId: itemId.threadId, namespace: previousIndex.id.namespace, id: previousIndex.id.id, key: self.sharedKey)
+                    if let previousValue = self.valueBox.get(self.table, key: previousKey) {
+                        if previousValue.length != 0 {
+                            info = previousValue
+                        }
+                    } else {
+                        assert(false)
+                    }
+                    self.valueBox.remove(self.table, key: previousKey, secure: true)
+                }
+                if let updatedInfo = self.updatedInfoItems[itemId] {
+                    if let value = updatedInfo.value {
+                        let encoder = PostboxEncoder()
+                        value.encode(encoder)
+                        info = encoder.makeReadBufferAndReset()
+                    } else {
+                        info = nil
+                    }
+                }
+                if info == nil {
+                    if let value = self.seedConfiguration.automaticThreadIndexInfo(itemId.peerId, itemId.threadId) {
+                        let encoder = PostboxEncoder()
+                        value.encode(encoder)
+                        info = encoder.makeReadBufferAndReset()
+                    }
+                }
+                
+                if let topIndex = topIndex, let info = info {
+                    if let previousIndex = previousIndex {
+                        self.reverseIndexTable.set(peerId: itemId.peerId, threadId: itemId.threadId, timestamp: previousIndex.timestamp, namespace: previousIndex.id.namespace, id: previousIndex.id.id, hasValue: false)
+                    }
+                    
+                    self.reverseIndexTable.set(peerId: itemId.peerId, threadId: itemId.threadId, timestamp: topIndex.timestamp, namespace: topIndex.id.namespace, id: topIndex.id.id, hasValue: true)
+                    self.valueBox.set(self.table, key: self.key(peerId: itemId.peerId, timestamp: topIndex.timestamp, threadId: itemId.threadId, namespace: topIndex.id.namespace, id: topIndex.id.id, key: self.sharedKey), value: info)
+                } else {
+                    if let previousIndex = previousIndex {
+                        self.reverseIndexTable.set(peerId: itemId.peerId, threadId: itemId.threadId, timestamp: previousIndex.timestamp, namespace: previousIndex.id.namespace, id: previousIndex.id.id, hasValue: false)
+                        self.valueBox.remove(self.table, key: self.key(peerId: itemId.peerId, timestamp: previousIndex.timestamp, threadId: itemId.threadId, namespace: previousIndex.id.namespace, id: previousIndex.id.id, key: self.sharedKey), secure: true)
+                    }
+                }
+            }
+        }
+        
+        return peerIds
+    }
+    
+    func fetch(peerId: PeerId, namespace: MessageId.Namespace, start: IndexBoundary, end: IndexBoundary, limit: Int) -> [(threadId: Int64, index: MessageIndex, info: StoredMessageHistoryThreadInfo)] {
+        let startKey: ValueBoxKey
+        switch start {
+        case let .index(index):
+            startKey = self.key(peerId: peerId, timestamp: index.timestamp, threadId: index.threadId, namespace: namespace, id: index.messageId, key: ValueBoxKey(length: self.sharedKey.length))
+        case .lowerBound:
+            startKey = self.lowerBound(peerId: peerId)
+        case .upperBound:
+            startKey = self.upperBound(peerId: peerId)
+        }
+        
+        let endKey: ValueBoxKey
+        switch end {
+        case let .index(index):
+            endKey = self.key(peerId: peerId, timestamp: index.timestamp, threadId: index.threadId, namespace: namespace, id: index.messageId, key: ValueBoxKey(length: self.sharedKey.length))
+        case .lowerBound:
+            endKey = self.lowerBound(peerId: peerId)
+        case .upperBound:
+            endKey = self.upperBound(peerId: peerId)
+        }
+        
+        var result: [(threadId: Int64, index: MessageIndex, info: StoredMessageHistoryThreadInfo)] = []
+        self.valueBox.range(self.table, start: startKey, end: endKey, values: { key, value in
+            let keyData = MessageHistoryThreadIndexTable.extract(key: key)
+            if value.length == 0 {
+                return true
+            }
+            let decoder = PostboxDecoder(buffer: value)
+            let state = StoredMessageHistoryThreadInfo(decoder: decoder)
+            result.append((keyData.threadId, keyData.index, state))
+            return true
+        }, limit: limit)
+        
+        return result
+    }
+    
+    func getAll(peerId: PeerId) -> [(threadId: Int64, index: MessageIndex, info: StoredMessageHistoryThreadInfo)] {
+        var result: [(threadId: Int64, index: MessageIndex, info: StoredMessageHistoryThreadInfo)] = []
+        self.valueBox.range(self.table, start: self.upperBound(peerId: peerId), end: self.lowerBound(peerId: peerId), values: { key, value in
+            let keyData = MessageHistoryThreadIndexTable.extract(key: key)
+            if value.length == 0 {
+                return true
+            }
+            let decoder = PostboxDecoder(buffer: value)
+            let state = StoredMessageHistoryThreadInfo(decoder: decoder)
+            result.append((keyData.threadId, keyData.index, state))
+            return true
+        }, limit: 100000)
+        
+        return result
+    }
+    
+    func getCount(peerId: PeerId) -> Int {
+        return self.valueBox.count(self.table, start: self.lowerBound(peerId: peerId), end: self.upperBound(peerId: peerId))
+    }
+    
+    override func beforeCommit() {
+        super.beforeCommit()
+        
+        self.updatedInfoItems.removeAll()
+    }
+}
+
+class MessageHistoryThreadPinnedTable: Table {
+    static func tableSpec(_ id: Int32) -> ValueBoxTable {
+        return ValueBoxTable(id: id, keyType: .binary, compactValuesOnCreation: true)
+    }
+    
+    private let sharedKey = ValueBoxKey(length: 8 + 4 + 8)
+    
+    override init(valueBox: ValueBox, table: ValueBoxTable, useCaches: Bool) {
+        super.init(valueBox: valueBox, table: table, useCaches: useCaches)
+    }
+    
+    private func key(peerId: PeerId, index: Int32, threadId: Int64) -> ValueBoxKey {
+        self.sharedKey.setInt64(0, value: peerId.toInt64())
+        self.sharedKey.setInt32(8, value: index)
+        self.sharedKey.setInt64(8 + 4, value: threadId)
+        
+        return self.sharedKey
+    }
+    
+    private static func extract(key: ValueBoxKey) -> (peerId: PeerId, threadId: Int64) {
+        return (
+            peerId: PeerId(key.getInt64(0)),
+            threadId: key.getInt64(8 + 4)
+        )
+    }
+    
+    private func lowerBound(peerId: PeerId) -> ValueBoxKey {
+        let key = ValueBoxKey(length: 8)
+        key.setInt64(0, value: peerId.toInt64())
+        return key
+    }
+    
+    private func upperBound(peerId: PeerId) -> ValueBoxKey {
+        return self.lowerBound(peerId: peerId).successor
+    }
+    
+    func get(peerId: PeerId) -> [Int64] {
+        var result: [Int64] = []
+        self.valueBox.range(self.table, start: self.lowerBound(peerId: peerId), end: self.upperBound(peerId: peerId), keys: { key in
+            result.append(MessageHistoryThreadPinnedTable.extract(key: key).threadId)
+            return true
+        }, limit: 0)
+        
+        return result
+    }
+    
+    func set(peerId: PeerId, threadIds: [Int64]) {
+        self.valueBox.removeRange(self.table, start: self.lowerBound(peerId: peerId), end: self.upperBound(peerId: peerId))
+        for i in 0 ..< threadIds.count {
+            self.valueBox.set(self.table, key: self.key(peerId: peerId, index: Int32(i), threadId: threadIds[i]), value: MemoryBuffer())
+        }
+    }
+    
+    override func beforeCommit() {
+        super.beforeCommit()
+    }
+}