CalvinBackup/ghostgram
1
0
Fork 0
mirror of https://github.com/ichmagmaus111/ghostgram.git synced 2026-06-09 03:23:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '7621e2f8dec938cf48181c8b10afc9b01f444e68' into beta

Browse Source
This commit is contained in:
Ilya Laktyushin
2025-12-06 02:17:48 +04:00
commit 8344b97e03
28070 changed files with 7995182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
submodules/TelegramCore/Sources/TelegramEngine/Resources/CollectCacheUsageStats.swift
+902
View File
@@ -0,0 +1,902 @@
import Foundation
import Postbox
import SwiftSignalKit
import MtProtoKit
import DarwinDirStat
public enum PeerCacheUsageCategory: Int32 {
case image = 0
case video
case audio
case file
}
public struct CacheUsageStats {
public let media: [PeerId: [PeerCacheUsageCategory: [MediaId: Int64]]]
public let mediaResourceIds: [MediaId: [MediaResourceId]]
public let peers: [PeerId: Peer]
public let otherSize: Int64
public let otherPaths: [String]
public let cacheSize: Int64
public let tempPaths: [String]
public let tempSize: Int64
public let immutableSize: Int64
public init(media: [PeerId: [PeerCacheUsageCategory: [MediaId: Int64]]], mediaResourceIds: [MediaId: [MediaResourceId]], peers: [PeerId: Peer], otherSize: Int64, otherPaths: [String], cacheSize: Int64, tempPaths: [String], tempSize: Int64, immutableSize: Int64) {
self.media = media
self.mediaResourceIds = mediaResourceIds
self.peers = peers
self.otherSize = otherSize
self.otherPaths = otherPaths
self.cacheSize = cacheSize
self.tempPaths = tempPaths
self.tempSize = tempSize
self.immutableSize = immutableSize
}
}
public enum CacheUsageStatsResult {
case progress(Float)
case result(CacheUsageStats)
}
private enum CollectCacheUsageStatsError {
case done(CacheUsageStats)
case generic
}
private final class CacheUsageStatsState {
var media: [PeerId: [PeerCacheUsageCategory: [MediaId: Int64]]] = [:]
var mediaResourceIds: [MediaId: [MediaResourceId]] = [:]
var allResourceIds = Set<MediaResourceId>()
var lowerBound: MessageIndex?
var upperBound: MessageIndex?
}
public final class StorageUsageStats {
public enum CategoryKey: Hashable {
case photos
case videos
case files
case music
case stickers
case avatars
case misc
case stories
}
public struct CategoryData {
public var size: Int64
public var messages: [EngineMessage.Id: Int64]
public init(size: Int64, messages: [EngineMessage.Id: Int64]) {
self.size = size
self.messages = messages
}
}
public fileprivate(set) var categories: [CategoryKey: CategoryData]
public init(categories: [CategoryKey: CategoryData]) {
self.categories = categories
}
}
public final class AllStorageUsageStats {
public final class PeerStats {
public let peer: EnginePeer
public let stats: StorageUsageStats
public init(peer: EnginePeer, stats: StorageUsageStats) {
self.peer = peer
self.stats = stats
}
}
public var deviceAvailableSpace: Int64
public var deviceFreeSpace: Int64
public fileprivate(set) var totalStats: StorageUsageStats
public fileprivate(set) var peers: [EnginePeer.Id: PeerStats]
public init(deviceAvailableSpace: Int64, deviceFreeSpace: Int64, totalStats: StorageUsageStats, peers: [EnginePeer.Id: PeerStats]) {
self.deviceAvailableSpace = deviceAvailableSpace
self.deviceFreeSpace = deviceFreeSpace
self.totalStats = totalStats
self.peers = peers
}
}
private extension StorageUsageStats {
convenience init(_ stats: StorageBox.Stats) {
var mappedCategories: [StorageUsageStats.CategoryKey: StorageUsageStats.CategoryData] = [:]
for (key, value) in stats.contentTypes {
let mappedCategory: StorageUsageStats.CategoryKey
switch key {
case MediaResourceUserContentType.image.rawValue:
mappedCategory = .photos
case MediaResourceUserContentType.video.rawValue:
mappedCategory = .videos
case MediaResourceUserContentType.file.rawValue:
mappedCategory = .files
case MediaResourceUserContentType.audio.rawValue:
mappedCategory = .music
case MediaResourceUserContentType.avatar.rawValue:
mappedCategory = .avatars
case MediaResourceUserContentType.sticker.rawValue:
mappedCategory = .stickers
case MediaResourceUserContentType.other.rawValue:
mappedCategory = .misc
case MediaResourceUserContentType.audioVideoMessage.rawValue:
mappedCategory = .misc
case MediaResourceUserContentType.story.rawValue:
mappedCategory = .stories
default:
mappedCategory = .misc
}
if mappedCategories[mappedCategory] == nil {
mappedCategories[mappedCategory] = StorageUsageStats.CategoryData(size: value.size, messages: value.messages)
} else {
mappedCategories[mappedCategory]?.size += value.size
mappedCategories[mappedCategory]?.messages.merge(value.messages, uniquingKeysWith: { lhs, _ in lhs})
}
}
self.init(categories: mappedCategories)
}
}
private func statForDirectory(path: String) -> Int64 {
if #available(macOS 10.13, *) {
var s = darwin_dirstat()
var result = dirstat_np(path, 1, &s, MemoryLayout<darwin_dirstat>.size)
if result != -1 {
return Int64(s.total_size)
} else {
result = dirstat_np(path, 0, &s, MemoryLayout<darwin_dirstat>.size)
if result != -1 {
return Int64(s.total_size)
} else {
return 0
}
}
} else {
let fileManager = FileManager.default
let folderURL = URL(fileURLWithPath: path)
var folderSize: Int64 = 0
if let files = try? fileManager.contentsOfDirectory(at: folderURL, includingPropertiesForKeys: nil, options: []) {
for file in files {
folderSize += (fileSize(file.path) ?? 0)
}
}
return folderSize
}
}
private func collectDirectoryUsageReportRecursive(path: String, indent: String, log: inout String) {
guard let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: path), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey], options: .skipsSubdirectoryDescendants) else {
return
}
for url in enumerator {
guard let url = url as? URL else {
continue
}
if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
let subdirectorySize = statForDirectory(path: url.path)
log.append("\(indent)+ \(url.lastPathComponent): \(subdirectorySize)\n")
collectDirectoryUsageReportRecursive(path: url.path, indent: indent + " ", log: &log)
} else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
log.append("\(indent)\(url.lastPathComponent): SYMLINK\n")
} else {
log.append("\(indent)\(url.lastPathComponent): \(fileSizeValue)\n")
}
}
}
}
public func collectRawStorageUsageReport(containerPath: String) -> String {
var log = ""
let documentsPath = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0]
let documentsSize = statForDirectory(path: documentsPath)
log.append("Documents (\(documentsPath)): \(documentsSize)\n")
collectDirectoryUsageReportRecursive(path: documentsPath, indent: " ", log: &log)
let systemCachePath = NSSearchPathForDirectoriesInDomains(.cachesDirectory, .userDomainMask, true)[0]
let systemCacheSize = statForDirectory(path: systemCachePath)
log.append("System Cache (\(systemCachePath)): \(systemCacheSize)\n")
let containerSize = statForDirectory(path: containerPath)
log.append("Container (\(containerPath)): \(containerSize)\n")
collectDirectoryUsageReportRecursive(path: containerPath, indent: " ", log: &log)
return log
}
func _internal_collectStorageUsageStats(account: Account) -> Signal<AllStorageUsageStats, NoError> {
let additionalStats = account.postbox.mediaBox.cacheStorageBox.totalSize() |> take(1)
return combineLatest(
additionalStats,
account.postbox.mediaBox.storageBox.getAllStats()
)
|> deliverOnMainQueue
|> mapToSignal { additionalStats, allStats -> Signal<AllStorageUsageStats, NoError> in
return account.postbox.transaction { transaction -> AllStorageUsageStats in
let total = StorageUsageStats(allStats.total)
if additionalStats != 0 {
if total.categories[.misc] == nil {
total.categories[.misc] = StorageUsageStats.CategoryData(size: 0, messages: [:])
}
total.categories[.misc]?.size += additionalStats
}
var peers: [EnginePeer.Id: AllStorageUsageStats.PeerStats] = [:]
for (peerId, peerStats) in allStats.peers {
if peerId.id._internalGetInt64Value() == 0 {
continue
}
var peerSize: Int64 = 0
for (_, contentValue) in peerStats.contentTypes {
peerSize += contentValue.size
}
if peerSize == 0 {
continue
}
if let peer = transaction.getPeer(peerId), transaction.getPeerChatListIndex(peerId) != nil {
peers[peerId] = AllStorageUsageStats.PeerStats(
peer: EnginePeer(peer),
stats: StorageUsageStats(peerStats)
)
}
}
let systemAttributes = try? FileManager.default.attributesOfFileSystem(forPath: NSHomeDirectory() as String)
let deviceAvailableSpace = (systemAttributes?[FileAttributeKey.systemSize] as? NSNumber)?.int64Value ?? 0
let deviceFreeSpace = (systemAttributes?[FileAttributeKey.systemFreeSize] as? NSNumber)?.int64Value ?? 0
return AllStorageUsageStats(
deviceAvailableSpace: deviceAvailableSpace,
deviceFreeSpace: deviceFreeSpace,
totalStats: total,
peers: peers
)
}
}
}
func _internal_renderStorageUsageStatsMessages(account: Account, stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey], existingMessages: [EngineMessage.Id: Message]) -> Signal<[EngineMessage.Id: Message], NoError> {
return account.postbox.transaction { transaction -> [EngineMessage.Id: Message] in
var result: [EngineMessage.Id: Message] = [:]
var peerInChatList: [EnginePeer.Id: Bool] = [:]
var messageIsHidden: [EngineMessage.Id: Bool] = [:]
for (category, value) in stats.categories {
if !categories.contains(category) {
continue
}
for (id, _) in value.messages.sorted(by: { $0.value >= $1.value }).prefix(1000) {
if result[id] == nil {
if let value = messageIsHidden[id] {
if value {
continue
}
}
if let message = existingMessages[id] {
if message.isSelfExpiring {
messageIsHidden[id] = true
} else {
messageIsHidden[id] = false
result[id] = message
}
} else {
var matches = false
if let peerInChatListValue = peerInChatList[id.peerId] {
if peerInChatListValue {
matches = true
}
} else {
let peerInChatListValue = transaction.getPeerChatListIndex(id.peerId) != nil
peerInChatList[id.peerId] = peerInChatListValue
if peerInChatListValue {
matches = true
}
}
if matches, let message = transaction.getMessage(id) {
if message.isSelfExpiring {
messageIsHidden[id] = true
} else {
messageIsHidden[id] = false
result[id] = message
}
}
}
}
}
}
return result
}
}
func _internal_clearStorage(account: Account, peerId: EnginePeer.Id?, categories: [StorageUsageStats.CategoryKey], includeMessages: [Message], excludeMessages: [Message]) -> Signal<Float, NoError> {
let mediaBox = account.postbox.mediaBox
return Signal { subscriber in
var includeResourceIds = Set<MediaResourceId>()
for message in includeMessages {
extractMediaResourceIds(message: message, resourceIds: &includeResourceIds)
}
var includeIds: [Data] = []
for resourceId in includeResourceIds {
if let data = resourceId.stringRepresentation.data(using: .utf8) {
includeIds.append(data)
}
}
var excludeResourceIds = Set<MediaResourceId>()
for message in excludeMessages {
extractMediaResourceIds(message: message, resourceIds: &excludeResourceIds)
}
var excludeIds: [Data] = []
for resourceId in excludeResourceIds {
if let data = resourceId.stringRepresentation.data(using: .utf8) {
excludeIds.append(data)
}
}
var mappedContentTypes: [UInt8] = []
for item in categories {
switch item {
case .photos:
mappedContentTypes.append(MediaResourceUserContentType.image.rawValue)
case .videos:
mappedContentTypes.append(MediaResourceUserContentType.video.rawValue)
case .files:
mappedContentTypes.append(MediaResourceUserContentType.file.rawValue)
case .music:
mappedContentTypes.append(MediaResourceUserContentType.audio.rawValue)
case .stickers:
mappedContentTypes.append(MediaResourceUserContentType.sticker.rawValue)
case .avatars:
mappedContentTypes.append(MediaResourceUserContentType.avatar.rawValue)
case .misc:
mappedContentTypes.append(MediaResourceUserContentType.other.rawValue)
mappedContentTypes.append(MediaResourceUserContentType.audioVideoMessage.rawValue)
// Legacy value for Gif
mappedContentTypes.append(5)
case .stories:
mappedContentTypes.append(MediaResourceUserContentType.story.rawValue)
}
}
mediaBox.storageBox.remove(peerId: peerId, contentTypes: mappedContentTypes, includeIds: includeIds, excludeIds: excludeIds, completion: { ids in
var resourceIds: [MediaResourceId] = []
for id in ids {
if let value = String(data: id, encoding: .utf8) {
resourceIds.append(MediaResourceId(value))
}
}
let _ = mediaBox.removeCachedResources(resourceIds).start(next: { progress in
subscriber.putNext(progress)
}, completed: {
if peerId == nil && categories.contains(.misc) {
let additionalPaths: [String] = [
"cache",
"animation-cache",
"short-cache",
]
for item in additionalPaths {
let fullPath = mediaBox.basePath + "/\(item)"
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: fullPath), includingPropertiesForKeys: [.isDirectoryKey], options: .skipsSubdirectoryDescendants) {
for url in enumerator {
guard let url = url as? URL else {
continue
}
let _ = try? FileManager.default.removeItem(at: url)
}
}
}
mediaBox.cacheStorageBox.reset()
subscriber.putCompletion()
} else {
subscriber.putCompletion()
}
})
})
return ActionDisposable {
}
}
}
func _internal_clearStorage(account: Account, peerIds: Set<EnginePeer.Id>, includeMessages: [Message], excludeMessages: [Message]) -> Signal<Float, NoError> {
let mediaBox = account.postbox.mediaBox
return Signal { subscriber in
var includeResourceIds = Set<MediaResourceId>()
for message in includeMessages {
extractMediaResourceIds(message: message, resourceIds: &includeResourceIds)
}
var includeIds: [Data] = []
for resourceId in includeResourceIds {
if let data = resourceId.stringRepresentation.data(using: .utf8) {
includeIds.append(data)
}
}
var excludeResourceIds = Set<MediaResourceId>()
for message in excludeMessages {
extractMediaResourceIds(message: message, resourceIds: &excludeResourceIds)
}
var excludeIds: [Data] = []
for resourceId in excludeResourceIds {
if let data = resourceId.stringRepresentation.data(using: .utf8) {
excludeIds.append(data)
}
}
mediaBox.storageBox.remove(peerIds: peerIds, includeIds: includeIds, excludeIds: excludeIds, completion: { ids in
var resourceIds: [MediaResourceId] = []
for id in ids {
if let value = String(data: id, encoding: .utf8) {
resourceIds.append(MediaResourceId(value))
}
}
let _ = mediaBox.removeCachedResources(resourceIds).start(next: { progress in
subscriber.putNext(progress)
}, completed: {
subscriber.putCompletion()
})
})
return ActionDisposable {
}
}
}
private func extractMediaResourceIds(message: Message, resourceIds: inout Set<MediaResourceId>) {
for media in message.media {
if let image = media as? TelegramMediaImage {
for representation in image.representations {
resourceIds.insert(representation.resource.id)
}
} else if let file = media as? TelegramMediaFile {
for representation in file.previewRepresentations {
resourceIds.insert(representation.resource.id)
}
resourceIds.insert(file.resource.id)
} else if let webpage = media as? TelegramMediaWebpage {
if case let .Loaded(content) = webpage.content {
if let image = content.image {
for representation in image.representations {
resourceIds.insert(representation.resource.id)
}
}
if let file = content.file {
for representation in file.previewRepresentations {
resourceIds.insert(representation.resource.id)
}
resourceIds.insert(file.resource.id)
}
}
} else if let game = media as? TelegramMediaGame {
if let image = game.image {
for representation in image.representations {
resourceIds.insert(representation.resource.id)
}
}
if let file = game.file {
for representation in file.previewRepresentations {
resourceIds.insert(representation.resource.id)
}
resourceIds.insert(file.resource.id)
}
}
}
}
func _internal_clearStorage(account: Account, messages: [Message]) -> Signal<Never, NoError> {
let mediaBox = account.postbox.mediaBox
return Signal { subscriber in
DispatchQueue.global().async {
var resourceIds = Set<MediaResourceId>()
for message in messages {
extractMediaResourceIds(message: message, resourceIds: &resourceIds)
}
var removeIds: [Data] = []
for resourceId in resourceIds {
if let id = resourceId.stringRepresentation.data(using: .utf8) {
removeIds.append(id)
}
}
mediaBox.storageBox.remove(ids: removeIds)
let _ = mediaBox.removeCachedResources(Array(resourceIds)).start(completed: {
subscriber.putCompletion()
})
}
return ActionDisposable {
}
}
}
func _internal_reindexCacheInBackground(account: Account, lowImpact: Bool) -> Signal<Never, NoError> {
let postbox = account.postbox
let queue = Queue(name: "ReindexCacheInBackground")
return Signal { subscriber in
let isCancelled = Atomic<Bool>(value: false)
func process(lowerBound: MessageIndex?) {
if isCancelled.with({ $0 }) {
return
}
let _ = (postbox.transaction { transaction -> (messagesByMediaId: [MediaId: [MessageId]], mediaMap: [MediaId: Media], nextLowerBound: MessageIndex?) in
return transaction.enumerateMediaMessages(lowerBound: lowerBound, upperBound: nil, limit: 1000)
}
|> deliverOn(queue)).start(next: { result in
Logger.shared.log("ReindexCacheInBackground", "process batch of \(result.mediaMap.count) media")
var storageItems: [(reference: StorageBox.Reference, id: Data, contentType: UInt8, size: Int64)] = []
let mediaBox = postbox.mediaBox
let processResource: ([MessageId], MediaResource, MediaResourceUserContentType) -> Void = { messageIds, resource, contentType in
let size = mediaBox.fileSizeForId(resource.id)
if size != 0 {
if let itemId = resource.id.stringRepresentation.data(using: .utf8) {
for messageId in messageIds {
storageItems.append((reference: StorageBox.Reference(peerId: messageId.peerId.toInt64(), messageNamespace: UInt8(clamping: messageId.namespace), messageId: messageId.id), id: itemId, contentType: contentType.rawValue, size: size))
}
}
}
}
for (_, media) in result.mediaMap {
guard let mediaId = media.id else {
continue
}
guard let mediaMessages = result.messagesByMediaId[mediaId] else {
continue
}
if let image = media as? TelegramMediaImage {
for representation in image.representations {
processResource(mediaMessages, representation.resource, .image)
}
} else if let file = media as? TelegramMediaFile {
for representation in file.previewRepresentations {
processResource(mediaMessages, representation.resource, MediaResourceUserContentType(file: file))
}
processResource(mediaMessages, file.resource, MediaResourceUserContentType(file: file))
for alternativeRepresentation in file.alternativeRepresentations {
for representation in alternativeRepresentation.previewRepresentations {
processResource(mediaMessages, representation.resource, MediaResourceUserContentType(file: file))
}
processResource(mediaMessages, alternativeRepresentation.resource, MediaResourceUserContentType(file: file))
}
} else if let webpage = media as? TelegramMediaWebpage {
if case let .Loaded(content) = webpage.content {
if let image = content.image {
for representation in image.representations {
processResource(mediaMessages, representation.resource, .image)
}
}
if let file = content.file {
for representation in file.previewRepresentations {
processResource(mediaMessages, representation.resource, MediaResourceUserContentType(file: file))
}
processResource(mediaMessages, file.resource, MediaResourceUserContentType(file: file))
}
}
} else if let game = media as? TelegramMediaGame {
if let image = game.image {
for representation in image.representations {
processResource(mediaMessages, representation.resource, .image)
}
}
if let file = game.file {
for representation in file.previewRepresentations {
processResource(mediaMessages, representation.resource, MediaResourceUserContentType(file: file))
}
processResource(mediaMessages, file.resource, MediaResourceUserContentType(file: file))
}
}
}
if !storageItems.isEmpty {
mediaBox.storageBox.batchAdd(items: storageItems)
}
if let nextLowerBound = result.nextLowerBound {
if lowImpact {
queue.after(0.4, {
process(lowerBound: nextLowerBound)
})
} else {
process(lowerBound: nextLowerBound)
}
} else {
subscriber.putCompletion()
}
})
}
process(lowerBound: nil)
return ActionDisposable {
let _ = isCancelled.swap(true)
}
}
|> runOn(queue)
}
func _internal_collectCacheUsageStats(account: Account, peerId: PeerId? = nil, additionalCachePaths: [String] = [], logFilesPath: String? = nil) -> Signal<CacheUsageStatsResult, NoError> {
return account.postbox.mediaBox.storageBox.all()
|> mapToSignal { entries -> Signal<CacheUsageStatsResult, NoError> in
final class IncrementalState {
var startIndex: Int = 0
var media: [PeerId: [PeerCacheUsageCategory: [MediaId: Int64]]] = [:]
var mediaResourceIds: [MediaId: [MediaResourceId]] = [:]
var totalSize: Int64 = 0
var mediaSize: Int64 = 0
var processedResourceIds = Set<String>()
var otherSize: Int64 = 0
var otherPaths: [String] = []
var peers: [PeerId: Peer] = [:]
}
let mediaBox = account.postbox.mediaBox
let queue = Queue()
return Signal<CacheUsageStatsResult, NoError> { subscriber in
var isCancelled: Bool = false
let state = Atomic<IncrementalState>(value: IncrementalState())
var processNextBatchPtr: (() -> Void)?
let processNextBatch: () -> Void = {
if isCancelled {
return
}
let _ = (account.postbox.transaction { transaction -> Void in
state.with { state in
if state.startIndex >= entries.count {
return
}
let batchCount = 5000
let endIndex = min(state.startIndex + batchCount, entries.count)
for i in state.startIndex ..< endIndex {
let entry = entries[i]
guard let resourceIdString = String(data: entry.id, encoding: .utf8) else {
continue
}
let resourceId = MediaResourceId(resourceIdString)
if state.processedResourceIds.contains(resourceId.stringRepresentation) {
continue
}
let resourceSize = mediaBox.resourceUsage(id: resourceId)
if resourceSize != 0 {
state.totalSize += resourceSize
for reference in entry.references {
if reference.peerId == 0 {
state.otherSize += resourceSize
let storePaths = mediaBox.storePathsForId(resourceId)
state.otherPaths.append(storePaths.complete)
state.otherPaths.append(storePaths.partial)
continue
}
if let message = transaction.getMessage(MessageId(peerId: PeerId(reference.peerId), namespace: MessageId.Namespace(reference.messageNamespace), id: reference.messageId)) {
for mediaItem in message.media {
guard let mediaId = mediaItem.id else {
continue
}
var category: PeerCacheUsageCategory?
if let _ = mediaItem as? TelegramMediaImage {
category = .image
} else if let mediaItem = mediaItem as? TelegramMediaFile {
if mediaItem.isMusic || mediaItem.isVoice {
category = .audio
} else if mediaItem.isVideo {
category = .video
} else {
category = .file
}
}
if let category = category {
state.mediaSize += resourceSize
state.processedResourceIds.insert(resourceId.stringRepresentation)
state.media[PeerId(reference.peerId), default: [:]][category, default: [:]][mediaId, default: 0] += resourceSize
if let index = state.mediaResourceIds.index(forKey: mediaId) {
if !state.mediaResourceIds[index].value.contains(resourceId) {
state.mediaResourceIds[mediaId]?.append(resourceId)
}
} else {
state.mediaResourceIds[mediaId] = [resourceId]
}
}
}
}
}
}
}
state.startIndex = endIndex
}
}).start(completed: {
if isCancelled {
return
}
let isFinished = state.with { state -> Bool in
return state.startIndex >= entries.count
}
if !isFinished {
queue.async {
processNextBatchPtr?()
}
} else {
let _ = (account.postbox.transaction { transaction -> Void in
state.with { state in
for peerId in state.media.keys {
if let peer = transaction.getPeer(peerId) {
state.peers[peer.id] = peer
}
}
}
}).start(completed: {
queue.async {
let state = state.with { $0 }
var tempPaths: [String] = []
var tempSize: Int64 = 0
#if os(iOS)
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: NSTemporaryDirectory()), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey]) {
for url in enumerator {
if let url = url as? URL {
if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
tempPaths.append(url.path)
} else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
tempPaths.append(url.path)
if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
} else {
tempSize += Int64(fileSizeValue)
}
}
}
}
}
#endif
var immutableSize: Int64 = 0
if let files = try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: account.basePath + "/postbox/db"), includingPropertiesForKeys: [URLResourceKey.fileSizeKey], options: []) {
for url in files {
if let fileSize = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize {
immutableSize += Int64(fileSize)
}
}
}
if let logFilesPath = logFilesPath, let files = try? FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: logFilesPath), includingPropertiesForKeys: [URLResourceKey.fileSizeKey], options: []) {
for url in files {
if let fileSize = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize {
immutableSize += Int64(fileSize)
}
}
}
for additionalPath in additionalCachePaths {
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: additionalPath), includingPropertiesForKeys: [.isDirectoryKey, .fileAllocatedSizeKey, .isSymbolicLinkKey]) {
for url in enumerator {
if let url = url as? URL {
if let isDirectoryValue = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectoryValue {
} else if let fileSizeValue = (try? url.resourceValues(forKeys: Set([.fileAllocatedSizeKey])))?.fileAllocatedSize {
tempPaths.append(url.path)
if let isSymbolicLinkValue = (try? url.resourceValues(forKeys: Set([.isSymbolicLinkKey])))?.isSymbolicLink, isSymbolicLinkValue {
} else {
tempSize += Int64(fileSizeValue)
}
}
}
}
}
}
var cacheSize: Int64 = 0
let basePath = account.postbox.mediaBox.basePath
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: basePath + "/cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
state.otherPaths.append("cache/" + url.lastPathComponent)
cacheSize += Int64(value)
}
}
}
}
func processRecursive(directoryPath: String, subdirectoryPath: String) {
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: directoryPath), includingPropertiesForKeys: [.fileSizeKey, .isDirectoryKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let isDirectory = (try? url.resourceValues(forKeys: Set([.isDirectoryKey])))?.isDirectory, isDirectory {
processRecursive(directoryPath: url.path, subdirectoryPath: subdirectoryPath + "/\(url.lastPathComponent)")
} else if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
state.otherPaths.append("\(subdirectoryPath)/" + url.lastPathComponent)
cacheSize += Int64(value)
}
}
}
}
}
processRecursive(directoryPath: basePath + "/animation-cache", subdirectoryPath: "animation-cache")
if let enumerator = FileManager.default.enumerator(at: URL(fileURLWithPath: basePath + "/short-cache"), includingPropertiesForKeys: [.fileSizeKey], options: [.skipsHiddenFiles, .skipsSubdirectoryDescendants], errorHandler: nil) {
loop: for url in enumerator {
if let url = url as? URL {
if let value = (try? url.resourceValues(forKeys: Set([.fileSizeKey])))?.fileSize, value != 0 {
state.otherPaths.append("short-cache/" + url.lastPathComponent)
cacheSize += Int64(value)
}
}
}
}
subscriber.putNext(.result(CacheUsageStats(
media: state.media,
mediaResourceIds: state.mediaResourceIds,
peers: state.peers,
otherSize: state.otherSize,
otherPaths: state.otherPaths,
cacheSize: cacheSize,
tempPaths: tempPaths,
tempSize: tempSize,
immutableSize: immutableSize
)))
subscriber.putCompletion()
}
})
}
})
}
processNextBatchPtr = {
processNextBatch()
}
processNextBatch()
return ActionDisposable {
isCancelled = true
}
}
|> runOn(queue)
}
}
func _internal_clearCachedMediaResources(account: Account, mediaResourceIds: Set<MediaResourceId>) -> Signal<Float, NoError> {
return account.postbox.mediaBox.removeCachedResources(Array(mediaResourceIds))
}
submodules/TelegramCore/Sources/TelegramEngine/Resources/TelegramEngineResources.swift
+419
View File
@@ -0,0 +1,419 @@
import Foundation
import SwiftSignalKit
import Postbox
import TelegramApi
public enum MediaResourceUserContentType: UInt8, Equatable {
case other = 0
case image = 1
case video = 2
case audio = 3
case file = 4
case sticker = 6
case avatar = 7
case audioVideoMessage = 8
case story = 9
}
public extension MediaResourceUserContentType {
init(file: TelegramMediaFile) {
if file.isInstantVideo || file.isVoice {
self = .audioVideoMessage
} else if file.isMusic {
self = .audio
} else if file.isSticker || file.isAnimatedSticker {
self = .sticker
} else if file.isCustomEmoji {
self = .sticker
} else if file.isVideo {
if file.isAnimated {
self = .other
} else {
self = .video
}
} else {
self = .file
}
}
}
public extension MediaResourceFetchParameters {
init(tag: MediaResourceFetchTag?, info: MediaResourceFetchInfo?, location: MediaResourceStorageLocation?, contentType: MediaResourceUserContentType, isRandomAccessAllowed: Bool) {
self.init(tag: tag, info: info, location: location, contentType: contentType.rawValue, isRandomAccessAllowed: isRandomAccessAllowed)
}
}
func bufferedFetch(_ signal: Signal<EngineMediaResource.Fetch.Result, EngineMediaResource.Fetch.Error>) -> Signal<EngineMediaResource.Fetch.Result, EngineMediaResource.Fetch.Error> {
return Signal { subscriber in
final class State {
var data = Data()
var isCompleted: Bool = false
init() {
}
}
let state = Atomic<State>(value: State())
return signal.start(next: { value in
switch value {
case let .dataPart(_, data, _, _):
let _ = state.with { state in
state.data.append(data)
}
case let .moveTempFile(file):
let _ = state.with { state in
state.isCompleted = true
}
subscriber.putNext(.moveTempFile(file: file))
case let .resourceSizeUpdated(size):
if size == 0 {
let _ = state.with { state in
state.data.removeAll()
state.isCompleted = true
}
let tempFile = TempBox.shared.tempFile(fileName: "file")
let _ = try? Data().write(to: URL(fileURLWithPath: tempFile.path), options: .atomic)
subscriber.putNext(.moveTempFile(file: tempFile))
subscriber.putCompletion()
}
default:
assert(false)
break
}
}, error: { error in
subscriber.putError(error)
}, completed: {
let tempFile = state.with { state -> TempBoxFile? in
if state.isCompleted {
return nil
} else {
let tempFile = TempBox.shared.tempFile(fileName: "data")
let _ = try? state.data.write(to: URL(fileURLWithPath: tempFile.path), options: .atomic)
return tempFile
}
}
if let tempFile = tempFile {
subscriber.putNext(.moveTempFile(file: tempFile))
}
subscriber.putCompletion()
})
}
}
public final class EngineMediaResource: Equatable {
public enum CacheTimeout {
case `default`
case shortLived
}
public struct ByteRange {
public enum Priority {
case `default`
case elevated
case maximum
}
public var range: Range<Int>
public var priority: Priority
public init(range: Range<Int>, priority: Priority) {
self.range = range
self.priority = priority
}
}
public final class Fetch {
public enum Result {
case dataPart(resourceOffset: Int64, data: Data, range: Range<Int64>, complete: Bool)
case resourceSizeUpdated(Int64)
case progressUpdated(Float)
case replaceHeader(data: Data, range: Range<Int64>)
case moveLocalFile(path: String)
case moveTempFile(file: TempBoxFile)
case copyLocalItem(MediaResourceDataFetchCopyLocalItem)
case reset
}
public enum Error {
case generic
}
public let signal: () -> Signal<Result, Error>
public init(_ signal: @escaping () -> Signal<Result, Error>) {
self.signal = signal
}
}
public final class ResourceData {
public let path: String
public let availableSize: Int64
public let isComplete: Bool
public init(
path: String,
availableSize: Int64,
isComplete: Bool
) {
self.path = path
self.availableSize = availableSize
self.isComplete = isComplete
}
}
public enum FetchStatus: Equatable {
case Remote(progress: Float)
case Local
case Fetching(isActive: Bool, progress: Float)
case Paused(progress: Float)
}
public struct Id: Equatable, Hashable {
public var stringRepresentation: String
public init(_ stringRepresentation: String) {
self.stringRepresentation = stringRepresentation
}
public init(_ id: MediaResourceId) {
self.stringRepresentation = id.stringRepresentation
}
}
private let resource: MediaResource
public init(_ resource: MediaResource) {
self.resource = resource
}
public func _asResource() -> MediaResource {
return self.resource
}
public var id: Id {
return Id(self.resource.id)
}
public static func ==(lhs: EngineMediaResource, rhs: EngineMediaResource) -> Bool {
return lhs.resource.isEqual(to: rhs.resource)
}
}
public extension EngineMediaResource.ResourceData {
convenience init(_ data: MediaResourceData) {
self.init(path: data.path, availableSize: data.size, isComplete: data.complete)
}
}
public extension EngineMediaResource.FetchStatus {
init(_ status: MediaResourceStatus) {
switch status {
case let .Remote(progress):
self = .Remote(progress: progress)
case .Local:
self = .Local
case let .Fetching(isActive, progress):
self = .Fetching(isActive: isActive, progress: progress)
case let .Paused(progress):
self = .Paused(progress: progress)
}
}
func _asStatus() -> MediaResourceStatus {
switch self {
case let .Remote(progress):
return .Remote(progress: progress)
case .Local:
return .Local
case let .Fetching(isActive, progress):
return .Fetching(isActive: isActive, progress: progress)
case let .Paused(progress):
return .Paused(progress: progress)
}
}
}
public extension TelegramEngine {
final class Resources {
private let account: Account
init(account: Account) {
self.account = account
}
public func preUpload(id: Int64, encrypt: Bool, tag: MediaResourceFetchTag?, source: Signal<EngineMediaResource.ResourceData, NoError>, onComplete: (()->Void)? = nil) {
return self.account.messageMediaPreuploadManager.add(network: self.account.network, postbox: self.account.postbox, id: id, encrypt: encrypt, tag: tag, source: source, onComplete: onComplete)
}
public func collectCacheUsageStats(peerId: PeerId? = nil, additionalCachePaths: [String] = [], logFilesPath: String? = nil) -> Signal<CacheUsageStatsResult, NoError> {
return _internal_collectCacheUsageStats(account: self.account, peerId: peerId, additionalCachePaths: additionalCachePaths, logFilesPath: logFilesPath)
}
public func collectStorageUsageStats() -> Signal<AllStorageUsageStats, NoError> {
return _internal_collectStorageUsageStats(account: self.account)
}
public func renderStorageUsageStatsMessages(stats: StorageUsageStats, categories: [StorageUsageStats.CategoryKey], existingMessages: [EngineMessage.Id: Message]) -> Signal<[EngineMessage.Id: Message], NoError> {
return _internal_renderStorageUsageStatsMessages(account: self.account, stats: stats, categories: categories, existingMessages: existingMessages)
}
public func clearStorage(peerId: EnginePeer.Id?, categories: [StorageUsageStats.CategoryKey], includeMessages: [Message], excludeMessages: [Message]) -> Signal<Float, NoError> {
return _internal_clearStorage(account: self.account, peerId: peerId, categories: categories, includeMessages: includeMessages, excludeMessages: excludeMessages)
}
public func clearStorage(peerIds: Set<EnginePeer.Id>, includeMessages: [Message], excludeMessages: [Message]) -> Signal<Float, NoError> {
_internal_clearStorage(account: self.account, peerIds: peerIds, includeMessages: includeMessages, excludeMessages: excludeMessages)
}
public func clearStorage(messages: [Message]) -> Signal<Never, NoError> {
_internal_clearStorage(account: self.account, messages: messages)
}
public func clearCachedMediaResources(mediaResourceIds: Set<MediaResourceId>) -> Signal<Float, NoError> {
return _internal_clearCachedMediaResources(account: self.account, mediaResourceIds: mediaResourceIds)
}
public func reindexCacheInBackground(lowImpact: Bool) -> Signal<Never, NoError> {
let mediaBox = self.account.postbox.mediaBox
return _internal_reindexCacheInBackground(account: self.account, lowImpact: lowImpact)
|> then(Signal { subscriber in
return mediaBox.updateResourceIndex(otherResourceContentType: MediaResourceUserContentType.other.rawValue, lowImpact: lowImpact, completion: {
subscriber.putCompletion()
})
})
}
public func data(id: EngineMediaResource.Id, attemptSynchronously: Bool = false) -> Signal<EngineMediaResource.ResourceData, NoError> {
return self.account.postbox.mediaBox.resourceData(
id: MediaResourceId(id.stringRepresentation),
pathExtension: nil,
option: .complete(waitUntilFetchStatus: false),
attemptSynchronously: attemptSynchronously
)
|> map { data in
return EngineMediaResource.ResourceData(data)
}
}
public func custom(
id: String,
fetch: EngineMediaResource.Fetch?,
cacheTimeout: EngineMediaResource.CacheTimeout = .default,
attemptSynchronously: Bool = false
) -> Signal<EngineMediaResource.ResourceData, NoError> {
let mappedKeepDuration: CachedMediaRepresentationKeepDuration
switch cacheTimeout {
case .default:
mappedKeepDuration = .general
case .shortLived:
mappedKeepDuration = .shortLived
}
return self.account.postbox.mediaBox.customResourceData(
id: id,
baseResourceId: nil,
pathExtension: nil,
complete: true,
fetch: fetch.flatMap { fetch in
return {
return Signal { subscriber in
return fetch.signal().start(next: { result in
let mappedResult: CachedMediaResourceRepresentationResult
switch result {
case let .moveTempFile(file):
mappedResult = .tempFile(file)
default:
assert(false)
return
}
subscriber.putNext(mappedResult)
}, completed: {
subscriber.putCompletion()
})
}
}
},
keepDuration: mappedKeepDuration,
attemptSynchronously: attemptSynchronously
)
|> map { data in
return EngineMediaResource.ResourceData(data)
}
}
public func httpData(url: String, preserveExactUrl: Bool = false) -> Signal<Data, EngineMediaResource.Fetch.Error> {
return fetchHttpResource(url: url, preserveExactUrl: preserveExactUrl)
|> mapError { _ -> EngineMediaResource.Fetch.Error in
return .generic
}
|> mapToSignal { value -> Signal<Data, EngineMediaResource.Fetch.Error> in
switch value {
case let .dataPart(_, data, _, _):
return .single(data)
default:
return .complete()
}
}
}
public func fetchAlbumCover(file: FileMediaReference?, title: String, performer: String, isThumbnail: Bool) -> Signal<EngineMediaResource.Fetch.Result, EngineMediaResource.Fetch.Error> {
let signal = currentWebDocumentsHostDatacenterId(postbox: self.account.postbox, isTestingEnvironment: self.account.testingEnvironment)
|> castError(EngineMediaResource.Fetch.Error.self)
|> take(1)
|> mapToSignal { datacenterId -> Signal<EngineMediaResource.Fetch.Result, EngineMediaResource.Fetch.Error> in
let resource = AlbumCoverResource(datacenterId: Int(datacenterId), file: file, title: title, performer: performer, isThumbnail: isThumbnail)
return multipartFetch(accountPeerId: self.account.peerId, postbox: self.account.postbox, network: self.account.network, mediaReferenceRevalidationContext: self.account.mediaReferenceRevalidationContext, networkStatsContext: self.account.networkStatsContext, resource: resource, datacenterId: Int(datacenterId), size: nil, intervals: .single([(0 ..< Int64.max, .default)]), parameters: MediaResourceFetchParameters(
tag: nil,
info: TelegramCloudMediaResourceFetchInfo(
reference: file?.resourceReference(resource) ?? MediaResourceReference.standalone(resource: resource),
preferBackgroundReferenceRevalidation: false,
continueInBackground: false
),
location: nil,
contentType: .image,
isRandomAccessAllowed: true
))
|> map { result -> EngineMediaResource.Fetch.Result in
switch result {
case let .dataPart(resourceOffset, data, range, complete):
return .dataPart(resourceOffset: resourceOffset, data: data, range: range, complete: complete)
case let .resourceSizeUpdated(size):
return .resourceSizeUpdated(size)
case let .progressUpdated(value):
return .progressUpdated(value)
case let .replaceHeader(data, range):
return .replaceHeader(data: data, range: range)
case let .moveLocalFile(path):
return .moveLocalFile(path: path)
case let .moveTempFile(file):
return .moveTempFile(file: file)
case let .copyLocalItem(item):
return .copyLocalItem(item)
case .reset:
return .reset
}
}
|> mapError { error -> EngineMediaResource.Fetch.Error in
switch error {
case .generic:
return .generic
}
}
}
return bufferedFetch(signal)
}
public func cancelAllFetches(id: String) {
preconditionFailure()
}
public func pushPriorityDownload(resourceId: String, priority: Int = 1) -> Disposable {
return self.account.network.multiplexedRequestManager.pushPriority(resourceId: resourceId, priority: priority)
}
public func applicationIcons() -> Signal<TelegramApplicationIcons, NoError> {
return _internal_applicationIcons(account: account)
}
}
}