Merge commit '7621e2f8dec938cf48181c8b10afc9b01f444e68' into beta

This commit is contained in:
Ilya Laktyushin
2025-12-06 02:17:48 +04:00
commit 8344b97e03
28070 changed files with 7995182 additions and 0 deletions
@@ -0,0 +1,81 @@
load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
load(
"@build_bazel_rules_apple//apple:resources.bzl",
"apple_resource_bundle",
"apple_resource_group",
)
load("//build-system/bazel-utils:plist_fragment.bzl",
"plist_fragment",
)
filegroup(
name = "MediaEditorMetalResources",
srcs = glob([
"MetalResources/**/*.*",
]),
visibility = ["//visibility:public"],
)
plist_fragment(
name = "MediaEditorBundleInfoPlist",
extension = "plist",
template =
"""
<key>CFBundleIdentifier</key>
<string>org.telegram.MediaEditor</string>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleName</key>
<string>MediaEditor</string>
"""
)
apple_resource_bundle(
name = "MediaEditorBundle",
infoplists = [
":MediaEditorBundleInfoPlist",
],
resources = [
":MediaEditorMetalResources",
],
)
swift_library(
name = "MediaEditor",
module_name = "MediaEditor",
srcs = glob([
"Sources/**/*.swift",
]),
copts = [
"-warnings-as-errors",
],
data = [
":MediaEditorBundle",
],
deps = [
"//submodules/AsyncDisplayKit",
"//submodules/Display",
"//submodules/Postbox",
"//submodules/TelegramCore",
"//submodules/SSignalKit/SwiftSignalKit",
"//submodules/TelegramPresentationData",
"//submodules/TelegramUIPreferences",
"//submodules/AccountContext",
"//submodules/AppBundle",
"//submodules/TextFormat",
"//submodules/AnimatedStickerNode",
"//submodules/TelegramAnimatedStickerNode",
"//submodules/StickerResources",
"//submodules/YuvConversion",
"//submodules/FastBlur",
"//submodules/WallpaperBackgroundNode",
"//submodules/ImageTransparency",
"//submodules/FFMpegBinding",
"//submodules/TelegramUI/Components/AnimationCache/ImageDCT",
"//submodules/FileMediaResourceStatus",
"//submodules/TelegramUI/Components/MediaEditor/ImageObjectSeparation",
],
visibility = [
"//visibility:public",
],
)
@@ -0,0 +1,28 @@
load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
swift_library(
name = "ImageObjectSeparation",
module_name = "ImageObjectSeparation",
srcs = glob([
"Sources/**/*.swift",
]),
copts = [
"-warnings-as-errors",
],
deps = [
"//submodules/AsyncDisplayKit",
"//submodules/Display",
"//submodules/Postbox",
"//submodules/TelegramCore",
"//submodules/SSignalKit/SwiftSignalKit",
"//submodules/AccountContext",
"//submodules/AppBundle",
"//submodules/ImageTransparency",
"//submodules/TelegramUI/Components/AnimationCache/ImageDCT",
"//submodules/FileMediaResourceStatus",
"//third-party/ZipArchive:ZipArchive",
],
visibility = [
"//visibility:public",
],
)
@@ -0,0 +1,436 @@
import Foundation
import UIKit
import Display
import Vision
import CoreImage
import CoreImage.CIFilterBuiltins
import VideoToolbox
import SwiftSignalKit
import Postbox
import TelegramCore
import AccountContext
import FileMediaResourceStatus
import ZipArchive
import ImageTransparency
private let queue = Queue()
public enum CutoutAvailability {
case available
case progress(Float)
case unavailable
}
private var forceCoreMLVariant: Bool {
#if targetEnvironment(simulator)
return true
#else
return false
#endif
}
private func modelPath() -> String {
return NSTemporaryDirectory() + "u2netp.mlmodelc"
}
public func cutoutAvailability(context: AccountContext) -> Signal<CutoutAvailability, NoError> {
if #available(iOS 17.0, *), !forceCoreMLVariant {
return .single(.available)
} else if #available(iOS 14.0, *) {
let compiledModelPath = modelPath()
if FileManager.default.fileExists(atPath: compiledModelPath) {
return .single(.available)
}
return context.engine.peers.resolvePeerByName(name: "stickersbackgroundseparation", referrer: nil)
|> mapToSignal { result -> Signal<CutoutAvailability, NoError> in
guard case let .result(maybePeer) = result else {
return .complete()
}
guard let peer = maybePeer else {
return .single(.unavailable)
}
return context.account.viewTracker.aroundMessageHistoryViewForLocation(.peer(peerId: peer.id, threadId: nil), index: .lowerBound, anchorIndex: .lowerBound, count: 5, fixedCombinedReadStates: nil)
|> mapToSignal { view -> Signal<(TelegramMediaFile, EngineMessage)?, NoError> in
if !view.0.isLoading {
if let message = view.0.entries.last?.message, let file = message.media.first(where: { $0 is TelegramMediaFile }) as? TelegramMediaFile {
return .single((file, EngineMessage(message)))
} else {
return .single(nil)
}
} else {
return .complete()
}
}
|> take(1)
|> mapToSignal { maybeFileAndMessage -> Signal<CutoutAvailability, NoError> in
if let (file, message) = maybeFileAndMessage {
let fetchedData = fetchedMediaResource(mediaBox: context.account.postbox.mediaBox, userLocation: .other, userContentType: .file, reference: FileMediaReference.message(message: MessageReference(message._asMessage()), media: file).resourceReference(file.resource))
enum FetchStatus {
case completed(String)
case progress(Float)
case failed
}
let fetchStatus = Signal<FetchStatus, NoError> { subscriber in
let fetchedDisposable = fetchedData.start()
let resourceDataDisposable = context.account.postbox.mediaBox.resourceData(file.resource, attemptSynchronously: false).start(next: { next in
if next.complete {
SSZipArchive.unzipFile(atPath: next.path, toDestination: NSTemporaryDirectory())
subscriber.putNext(.completed(compiledModelPath))
subscriber.putCompletion()
}
}, error: subscriber.putError, completed: subscriber.putCompletion)
let progressDisposable = messageFileMediaResourceStatus(context: context, file: file, message: message, isRecentActions: false).start(next: { status in
switch status.fetchStatus {
case let .Remote(progress), let .Fetching(_, progress), let .Paused(progress):
subscriber.putNext(.progress(progress))
default:
break
}
})
return ActionDisposable {
fetchedDisposable.dispose()
resourceDataDisposable.dispose()
progressDisposable.dispose()
}
}
return fetchStatus
|> mapToSignal { status -> Signal<CutoutAvailability, NoError> in
switch status {
case .completed:
return .single(.available)
case let .progress(progress):
return .single(.progress(progress))
case .failed:
return .single(.unavailable)
}
}
} else {
return .single(.unavailable)
}
}
}
} else {
return .single(.unavailable)
}
}
public func cutoutStickerImage(from image: UIImage, context: AccountContext? = nil, onlyCheck: Bool = false) -> Signal<UIImage?, NoError> {
guard let cgImage = image.cgImage else {
return .single(nil)
}
if #available(iOS 17.0, *), !forceCoreMLVariant {
return Signal { subscriber in
let ciContext = CIContext(options: nil)
let inputImage = CIImage(cgImage: cgImage)
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
let request = VNGenerateForegroundInstanceMaskRequest { [weak handler] request, error in
guard let handler, let result = request.results?.first as? VNInstanceMaskObservation else {
subscriber.putNext(nil)
subscriber.putCompletion()
return
}
if onlyCheck {
subscriber.putNext(UIImage())
subscriber.putCompletion()
} else {
let instances = instances(atPoint: nil, inObservation: result)
if let mask = try? result.generateScaledMaskForImage(forInstances: instances, from: handler) {
let filter = CIFilter.blendWithMask()
filter.inputImage = inputImage
filter.backgroundImage = CIImage(color: .clear)
filter.maskImage = CIImage(cvPixelBuffer: mask)
if let output = filter.outputImage, let cgImage = ciContext.createCGImage(output, from: inputImage.extent) {
let image = UIImage(cgImage: cgImage)
subscriber.putNext(image)
subscriber.putCompletion()
return
}
}
subscriber.putNext(nil)
subscriber.putCompletion()
}
}
try? handler.perform([request])
return ActionDisposable {
request.cancel()
}
}
|> runOn(queue)
} else if #available(iOS 14.0, *), onlyCheck {
return Signal { subscriber in
U2netp.load(contentsOf: URL(fileURLWithPath: modelPath()), completionHandler: { result in
switch result {
case let .success(model):
let modelImageSize = CGSize(width: 320, height: 320)
if let squareImage = scaleImageToPixelSize(image: image, size: modelImageSize),
let pixelBuffer = buffer(from: squareImage),
let result = try? model.prediction(in_0: pixelBuffer),
let resultImage = UIImage(pixelBuffer: result.out_p1),
imageHasSubject(resultImage) {
subscriber.putNext(UIImage())
} else {
subscriber.putNext(nil)
}
subscriber.putCompletion()
case .failure:
subscriber.putNext(nil)
subscriber.putCompletion()
}
})
return EmptyDisposable
}
|> runOn(queue)
} else {
return .single(nil)
}
}
public struct CutoutResult {
public enum Image {
case image(UIImage, CIImage)
case pixelBuffer(CVPixelBuffer)
}
public let index: Int
public let extractedImage: Image?
public let edgesMaskImage: Image?
public let maskImage: Image?
public let backgroundImage: Image?
}
public enum CutoutTarget {
case point(CGPoint?)
case index(Int)
case all
}
func refineEdges(_ maskImage: CIImage) -> CIImage? {
let maskImage = maskImage.clampedToExtent()
let blurFilter = CIFilter(name: "CIGaussianBlur")!
blurFilter.setValue(maskImage, forKey: kCIInputImageKey)
blurFilter.setValue(11.4, forKey: kCIInputRadiusKey)
let controlsFilter = CIFilter(name: "CIColorControls")!
controlsFilter.setValue(blurFilter.outputImage, forKey: kCIInputImageKey)
controlsFilter.setValue(6.61, forKey: kCIInputContrastKey)
let sharpenFilter = CIFilter(name: "CISharpenLuminance")!
sharpenFilter.setValue(controlsFilter.outputImage, forKey: kCIInputImageKey)
sharpenFilter.setValue(250.0, forKey: kCIInputSharpnessKey)
return sharpenFilter.outputImage?.cropped(to: maskImage.extent)
}
public func cutoutImage(
from image: UIImage,
editedImage: UIImage? = nil,
crop: (offset: CGPoint, rotation: CGFloat, scale: CGFloat)?,
target: CutoutTarget,
includeExtracted: Bool = true,
completion: @escaping ([CutoutResult]) -> Void
) {
guard #available(iOS 14.0, *), let cgImage = image.cgImage else {
completion([])
return
}
let ciContext = CIContext(options: nil)
let inputImage = CIImage(cgImage: cgImage)
var results: [CutoutResult] = []
func process(instance: Int, mask originalMaskImage: CIImage) {
let extractedImage: CutoutResult.Image?
if includeExtracted {
let filter = CIFilter.blendWithMask()
filter.backgroundImage = CIImage(color: .clear)
let dimensions: CGSize
var maskImage = originalMaskImage
if let editedImage = editedImage?.cgImage.flatMap({ CIImage(cgImage: $0) }) {
filter.inputImage = editedImage
dimensions = editedImage.extent.size
if let (cropOffset, cropRotation, cropScale) = crop {
let initialScale: CGFloat
if maskImage.extent.height > maskImage.extent.width {
initialScale = dimensions.width / maskImage.extent.width
} else {
initialScale = dimensions.width / maskImage.extent.height
}
let dimensions = editedImage.extent.size
maskImage = maskImage.transformed(by: CGAffineTransform(translationX: -maskImage.extent.width / 2.0, y: -maskImage.extent.height / 2.0))
var transform = CGAffineTransform.identity
transform = transform.translatedBy(x: dimensions.width / 2.0 + cropOffset.x, y: dimensions.height / 2.0 + cropOffset.y * -1.0)
transform = transform.rotated(by: -cropRotation)
transform = transform.scaledBy(x: cropScale * initialScale, y: cropScale * initialScale)
maskImage = maskImage.transformed(by: transform)
}
} else {
filter.inputImage = inputImage
dimensions = inputImage.extent.size
}
filter.maskImage = maskImage
if let output = filter.outputImage, let cgImage = ciContext.createCGImage(output, from: CGRect(origin: .zero, size: dimensions)) {
extractedImage = .image(UIImage(cgImage: cgImage), output)
} else {
extractedImage = nil
}
} else {
extractedImage = nil
}
let whiteImage = CIImage(color: .white)
let blackImage = CIImage(color: .black)
let maskFilter = CIFilter.blendWithMask()
maskFilter.inputImage = whiteImage
maskFilter.backgroundImage = blackImage
maskFilter.maskImage = originalMaskImage
let refinedMaskFilter = CIFilter.blendWithMask()
refinedMaskFilter.inputImage = whiteImage
refinedMaskFilter.backgroundImage = blackImage
refinedMaskFilter.maskImage = refineEdges(originalMaskImage)
let edgesMaskImage: CutoutResult.Image?
let maskImage: CutoutResult.Image?
if let maskOutput = maskFilter.outputImage?.cropped(to: inputImage.extent), let maskCgImage = ciContext.createCGImage(maskOutput, from: inputImage.extent), let refinedMaskOutput = refinedMaskFilter.outputImage?.cropped(to: inputImage.extent), let refinedMaskCgImage = ciContext.createCGImage(refinedMaskOutput, from: inputImage.extent) {
edgesMaskImage = .image(UIImage(cgImage: maskCgImage), maskOutput)
maskImage = .image(UIImage(cgImage: refinedMaskCgImage), refinedMaskOutput)
} else {
edgesMaskImage = nil
maskImage = nil
}
if extractedImage != nil || maskImage != nil {
results.append(CutoutResult(index: instance, extractedImage: extractedImage, edgesMaskImage: edgesMaskImage, maskImage: maskImage, backgroundImage: nil))
}
}
if #available(iOS 17.0, *), !forceCoreMLVariant {
queue.async {
let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
let request = VNGenerateForegroundInstanceMaskRequest { [weak handler] request, error in
guard let handler, let result = request.results?.first as? VNInstanceMaskObservation else {
completion([])
return
}
let targetInstances: IndexSet
switch target {
case let .point(point):
targetInstances = instances(atPoint: point, inObservation: result)
case let .index(index):
targetInstances = IndexSet([index])
case .all:
targetInstances = result.allInstances
}
for instance in targetInstances {
if let mask = try? result.generateScaledMaskForImage(forInstances: IndexSet(integer: instance), from: handler) {
process(instance: instance, mask: CIImage(cvPixelBuffer: mask))
}
}
completion(results)
}
try? handler.perform([request])
}
} else {
U2netp.load(contentsOf: URL(fileURLWithPath: modelPath()), completionHandler: { result in
switch result {
case let .success(model):
let modelImageSize = CGSize(width: 320, height: 320)
if let squareImage = scaleImageToPixelSize(image: image, size: modelImageSize), let pixelBuffer = buffer(from: squareImage), let result = try? model.prediction(in_0: pixelBuffer), let maskImage = UIImage(pixelBuffer: result.out_p1), let scaledMaskImage = scaleImageToPixelSize(image: maskImage, size: image.size), let ciImage = CIImage(image: scaledMaskImage) {
process(instance: 0, mask: ciImage)
}
case .failure:
break
}
completion(results)
})
}
}
@available(iOS 17.0, *)
private func instances(atPoint maybePoint: CGPoint?, inObservation observation: VNInstanceMaskObservation) -> IndexSet {
guard let point = maybePoint else {
return observation.allInstances
}
let instanceMap = observation.instanceMask
let coords = VNImagePointForNormalizedPoint(point, CVPixelBufferGetWidth(instanceMap) - 1, CVPixelBufferGetHeight(instanceMap) - 1)
CVPixelBufferLockBaseAddress(instanceMap, .readOnly)
guard let pixels = CVPixelBufferGetBaseAddress(instanceMap) else {
fatalError()
}
let bytesPerRow = CVPixelBufferGetBytesPerRow(instanceMap)
let instanceLabel = pixels.load(fromByteOffset: Int(coords.y) * bytesPerRow + Int(coords.x), as: UInt8.self)
CVPixelBufferUnlockBaseAddress(instanceMap, .readOnly)
return instanceLabel == 0 ? observation.allInstances : [Int(instanceLabel)]
}
private extension UIImage {
convenience init?(pixelBuffer: CVPixelBuffer) {
var cgImage: CGImage?
VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
guard let cgImage = cgImage else {
return nil
}
self.init(cgImage: cgImage)
}
}
private func scaleImageToPixelSize(image: UIImage, size: CGSize) -> UIImage? {
UIGraphicsBeginImageContextWithOptions(size, true, 1.0)
image.draw(in: CGRect(origin: CGPoint(), size: size), blendMode: .copy, alpha: 1.0)
let result = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
return result
}
private func buffer(from image: UIImage) -> CVPixelBuffer? {
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
var pixelBuffer : CVPixelBuffer?
let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(image.size.width), Int(image.size.height), kCVPixelFormatType_32ARGB, attrs, &pixelBuffer)
guard (status == kCVReturnSuccess) else {
return nil
}
guard let pixelBufferUnwrapped = pixelBuffer else {
return nil
}
CVPixelBufferLockBaseAddress(pixelBufferUnwrapped, CVPixelBufferLockFlags(rawValue: 0))
let pixelData = CVPixelBufferGetBaseAddress(pixelBufferUnwrapped)
let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(data: pixelData, width: Int(image.size.width), height: Int(image.size.height), bitsPerComponent: 8, bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBufferUnwrapped), space: rgbColorSpace, bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue) else {
return nil
}
context.translateBy(x: 0, y: image.size.height)
context.scaleBy(x: 1.0, y: -1.0)
UIGraphicsPushContext(context)
image.draw(in: CGRect(x: 0, y: 0, width: image.size.width, height: image.size.height))
UIGraphicsPopContext()
CVPixelBufferUnlockBaseAddress(pixelBufferUnwrapped, CVPixelBufferLockFlags(rawValue: 0))
return pixelBufferUnwrapped
}
@@ -0,0 +1,252 @@
import CoreML
/// Model Prediction Input Type
@available(macOS 13.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)
class U2netpInput : MLFeatureProvider {
/// in_0 as color (kCVPixelFormatType_32BGRA) image buffer, 320 pixels wide by 320 pixels high
var in_0: CVPixelBuffer
var featureNames: Set<String> {
get {
return ["in_0"]
}
}
func featureValue(for featureName: String) -> MLFeatureValue? {
if (featureName == "in_0") {
return MLFeatureValue(pixelBuffer: in_0)
}
return nil
}
init(in_0: CVPixelBuffer) {
self.in_0 = in_0
}
convenience init(in_0With in_0: CGImage) throws {
self.init(in_0: try MLFeatureValue(cgImage: in_0, pixelsWide: 320, pixelsHigh: 320, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!)
}
convenience init(in_0At in_0: URL) throws {
self.init(in_0: try MLFeatureValue(imageAt: in_0, pixelsWide: 320, pixelsHigh: 320, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!)
}
func setIn_0(with in_0: CGImage) throws {
self.in_0 = try MLFeatureValue(cgImage: in_0, pixelsWide: 320, pixelsHigh: 320, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!
}
func setIn_0(with in_0: URL) throws {
self.in_0 = try MLFeatureValue(imageAt: in_0, pixelsWide: 320, pixelsHigh: 320, pixelFormatType: kCVPixelFormatType_32ARGB, options: nil).imageBufferValue!
}
}
/// Model Prediction Output Type
@available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)
class U2netpOutput : MLFeatureProvider {
/// Source provided by CoreML
private let provider : MLFeatureProvider
/// out_p0 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p0: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p0")!.imageBufferValue
}()!
/// out_p1 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p1: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p1")!.imageBufferValue
}()!
/// out_p2 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p2: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p2")!.imageBufferValue
}()!
/// out_p3 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p3: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p3")!.imageBufferValue
}()!
/// out_p4 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p4: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p4")!.imageBufferValue
}()!
/// out_p5 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p5: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p5")!.imageBufferValue
}()!
/// out_p6 as grayscale (kCVPixelFormatType_OneComponent8) image buffer, 320 pixels wide by 320 pixels high
lazy var out_p6: CVPixelBuffer = {
[unowned self] in return self.provider.featureValue(for: "out_p6")!.imageBufferValue
}()!
var featureNames: Set<String> {
return self.provider.featureNames
}
func featureValue(for featureName: String) -> MLFeatureValue? {
return self.provider.featureValue(for: featureName)
}
init(out_p0: CVPixelBuffer, out_p1: CVPixelBuffer, out_p2: CVPixelBuffer, out_p3: CVPixelBuffer, out_p4: CVPixelBuffer, out_p5: CVPixelBuffer, out_p6: CVPixelBuffer) {
self.provider = try! MLDictionaryFeatureProvider(dictionary: ["out_p0" : MLFeatureValue(pixelBuffer: out_p0), "out_p1" : MLFeatureValue(pixelBuffer: out_p1), "out_p2" : MLFeatureValue(pixelBuffer: out_p2), "out_p3" : MLFeatureValue(pixelBuffer: out_p3), "out_p4" : MLFeatureValue(pixelBuffer: out_p4), "out_p5" : MLFeatureValue(pixelBuffer: out_p5), "out_p6" : MLFeatureValue(pixelBuffer: out_p6)])
}
init(features: MLFeatureProvider) {
self.provider = features
}
}
/// Class for model loading and prediction
@available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)
class U2netp {
let model: MLModel
/**
Construct U2netp instance with an existing MLModel object.
Usually the application does not use this initializer unless it makes a subclass of U2netp.
Such application may want to use `MLModel(contentsOfURL:configuration:)` and `U2netp.urlOfModelInThisBundle` to create a MLModel object to pass-in.
- parameters:
- model: MLModel object
*/
init(model: MLModel) {
self.model = model
}
/**
Construct U2netp instance with explicit path to mlmodelc file
- parameters:
- modelURL: the file url of the model
- throws: an NSError object that describes the problem
*/
convenience init(contentsOf modelURL: URL) throws {
try self.init(model: MLModel(contentsOf: modelURL))
}
/**
Construct a model with URL of the .mlmodelc directory and configuration
- parameters:
- modelURL: the file url of the model
- configuration: the desired model configuration
- throws: an NSError object that describes the problem
*/
convenience init(contentsOf modelURL: URL, configuration: MLModelConfiguration) throws {
try self.init(model: MLModel(contentsOf: modelURL, configuration: configuration))
}
/**
Construct U2netp instance asynchronously with URL of the .mlmodelc directory with optional configuration.
Model loading may take time when the model content is not immediately available (e.g. encrypted model). Use this factory method especially when the caller is on the main thread.
- parameters:
- modelURL: the URL to the model
- configuration: the desired model configuration
- handler: the completion handler to be called when the model loading completes successfully or unsuccessfully
*/
class func load(contentsOf modelURL: URL, configuration: MLModelConfiguration = MLModelConfiguration(), completionHandler handler: @escaping (Swift.Result<U2netp, Error>) -> Void) {
MLModel.load(contentsOf: modelURL, configuration: configuration) { result in
switch result {
case .failure(let error):
handler(.failure(error))
case .success(let model):
handler(.success(U2netp(model: model)))
}
}
}
/**
Construct U2netp instance asynchronously with URL of the .mlmodelc directory with optional configuration.
Model loading may take time when the model content is not immediately available (e.g. encrypted model). Use this factory method especially when the caller is on the main thread.
- parameters:
- modelURL: the URL to the model
- configuration: the desired model configuration
*/
@available(macOS 12.0, iOS 15.0, tvOS 15.0, watchOS 8.0, *)
class func load(contentsOf modelURL: URL, configuration: MLModelConfiguration = MLModelConfiguration()) async throws -> U2netp {
let model = try await MLModel.load(contentsOf: modelURL, configuration: configuration)
return U2netp(model: model)
}
/**
Make a prediction using the structured interface
- parameters:
- input: the input to the prediction as U2netpInput
- throws: an NSError object that describes the problem
- returns: the result of the prediction as U2netpOutput
*/
func prediction(input: U2netpInput) throws -> U2netpOutput {
return try self.prediction(input: input, options: MLPredictionOptions())
}
/**
Make a prediction using the structured interface
- parameters:
- input: the input to the prediction as U2netpInput
- options: prediction options
- throws: an NSError object that describes the problem
- returns: the result of the prediction as U2netpOutput
*/
func prediction(input: U2netpInput, options: MLPredictionOptions) throws -> U2netpOutput {
let outFeatures = try model.prediction(from: input, options:options)
return U2netpOutput(features: outFeatures)
}
/**
Make a prediction using the convenience interface
- parameters:
- in_0 as color (kCVPixelFormatType_32BGRA) image buffer, 320 pixels wide by 320 pixels high
- throws: an NSError object that describes the problem
- returns: the result of the prediction as U2netpOutput
*/
func prediction(in_0: CVPixelBuffer) throws -> U2netpOutput {
let input_ = U2netpInput(in_0: in_0)
return try self.prediction(input: input_)
}
/**
Make a batch prediction using the structured interface
- parameters:
- inputs: the inputs to the prediction as [U2netpInput]
- options: prediction options
- throws: an NSError object that describes the problem
- returns: the result of the prediction as [U2netpOutput]
*/
func predictions(inputs: [U2netpInput], options: MLPredictionOptions = MLPredictionOptions()) throws -> [U2netpOutput] {
let batchIn = MLArrayBatchProvider(array: inputs)
let batchOut = try model.predictions(from: batchIn, options: options)
var results : [U2netpOutput] = []
results.reserveCapacity(inputs.count)
for i in 0..<batchOut.count {
let outProvider = batchOut.features(at: i)
let result = U2netpOutput(features: outProvider)
results.append(result)
}
return results
}
}
@@ -0,0 +1,204 @@
#include <metal_stdlib>
#include "EditorCommon.h"
#include "EditorUtils.h"
using namespace metal;
typedef struct {
float2 dimensions;
float aspectRatio;
float shadows;
float highlights;
float contrast;
float fade;
float saturation;
float shadowsTintIntensity;
float3 shadowsTintColor;
float highlightsTintIntensity;
float3 highlightsTintColor;
float exposure;
float warmth;
float grain;
float vignette;
float hasCurves;
float2 empty;
} MediaEditorAdjustments;
half3 fade(half3 color, float fadeAmount) {
half3 comp1 = half3(-0.9772) * half3(pow(float3(color), float3(3.0)));
half3 comp2 = half3(1.708) * half3(pow(float3(color), float3(2.0)));
half3 comp3 = half3(-0.1603) * color;
half3 comp4 = half3(0.2878);
half3 finalComponent = comp1 + comp2 + comp3 + comp4;
half3 difference = finalComponent - color;
half3 scalingValue = half3(0.9);
half3 faded = color + (difference * scalingValue);
return (color * (1.0 - fadeAmount)) + (faded * fadeAmount);
}
float3 tintRaiseShadowsCurve(half3 color) {
float3 comp1 = float3(-0.003671) * pow(float3(color), float3(3.0));
float3 comp2 = float3(0.3842) * pow(float3(color), float3(2.0));
float3 comp3 = float3(0.3764) * float3(color);
float3 comp4 = float3(0.2515);
return comp1 + comp2 + comp3 + comp4;
}
half3 tintShadows(half3 color, float3 tintColor, float tintAmount) {
float3 raisedShadows = tintRaiseShadowsCurve(color);
float3 tintedShadows = mix(float3(color), raisedShadows, tintColor);
float3 tintedShadowsWithAmount = mix(float3(color), tintedShadows, tintAmount);
return half3(clamp(tintedShadowsWithAmount, 0.0, 1.0));
}
half3 tintHighlights(half3 color, float3 tintColor, float tintAmount) {
float3 loweredHighlights = float3(1.0) - tintRaiseShadowsCurve(half3(1.0) - color);
float3 tintedHighlights = mix(float3(color), loweredHighlights, (float3(1.0) - tintColor));
float3 tintedHighlightsWithAmount = mix(float3(color), tintedHighlights, tintAmount);
return half3(clamp(tintedHighlightsWithAmount, 0.0, 1.0));
}
half3 applyLuminanceCurve(half3 pixel, constant float allCurve[200]) {
int index = int(clamp(pixel.z / (1.0 / 200.0), 0.0, 199.0));
float value = allCurve[index];
float grayscale = (smoothstep(0.0, 0.1, float(pixel.z)) * (1.0 - smoothstep(0.8, 1.0, float(pixel.z))));
half saturation = mix(0.0, float(pixel.y), grayscale);
pixel.y = saturation;
pixel.z = value;
return pixel;
}
half3 applyRGBCurve(half3 pixel, constant float redCurve[200], constant float greenCurve[200], constant float blueCurve[200]) {
int index = int(clamp(pixel.r / (1.0 / 200.0), 0.0, 199.0));
float value = redCurve[index];
pixel.r = value;
index = int(clamp(pixel.g / (1.0 / 200.0), 0.0, 199.0));
value = greenCurve[index];
pixel.g = clamp(value, 0.0, 1.0);
index = int(clamp(pixel.b / (1.0 / 200.0), 0.0, 199.0));
value = blueCurve[index];
pixel.b = clamp(value, 0.0, 1.0);
return pixel;
}
fragment half4 adjustmentsFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> sourceImage [[texture(0)]],
constant MediaEditorAdjustments& adjustments [[buffer(0)]],
constant float allCurve [[buffer(1)]][200],
constant float redCurve [[buffer(2)]][200],
constant float greenCurve [[buffer(3)]][200],
constant float blueCurve [[buffer(4)]][200]
) {
constexpr sampler samplr(filter::linear, mag_filter::linear, min_filter::linear);
const float epsilon = 0.005;
half4 source = sourceImage.sample(samplr, float2(in.texCoord.x, in.texCoord.y));
half4 result = source;
if (adjustments.hasCurves > epsilon) {
result = half4(applyRGBCurve(hslToRgb(applyLuminanceCurve(rgbToHsl(result.rgb), allCurve)), redCurve, greenCurve, blueCurve), result.a);
}
if (abs(adjustments.highlights) > epsilon || abs(adjustments.shadows) > epsilon) {
const float3 hsLuminanceWeighting = float3(0.3, 0.3, 0.3);
float mappedHighlights = adjustments.highlights * 0.75 + 1.0;
float mappedShadows = adjustments.shadows * 0.55 + 1.0;
float hsLuminance = dot(float3(result.rgb), hsLuminanceWeighting);
float shadow = clamp((pow(hsLuminance, 1.0 / mappedShadows) - 0.76 * pow(hsLuminance, 2.0 / mappedShadows)) - hsLuminance, 0.0, 1.0);
float highlight = clamp((1.0 - (pow(1.0 - hsLuminance, 1.0 / (2.0 - mappedHighlights)) - 0.8 * pow(1.0 - hsLuminance, 2.0 / (2.0 - mappedHighlights)))) - hsLuminance, -1.0, 0.0);
float3 hsResult = float3(0.0, 0.0, 0.0) + ((hsLuminance + shadow + highlight) - 0.0) * ((float3(result.rgb) - float3(0.0, 0.0, 0.0)) / (hsLuminance - 0.0));
float contrastedLuminance = ((hsLuminance - 0.5) * 1.5) + 0.5;
float whiteInterp = contrastedLuminance * contrastedLuminance * contrastedLuminance;
half whiteTarget = clamp(mappedHighlights, 1.0, 2.0) - 1.0;
hsResult = mix(hsResult, float3(1.0), whiteInterp * whiteTarget);
float invContrastedLuminance = 1.0 - contrastedLuminance;
float blackInterp = invContrastedLuminance * invContrastedLuminance * invContrastedLuminance;
half blackTarget = 1.0 - clamp(mappedShadows, 0.0, 1.0);
result.rgb = half3(mix(hsResult, float3(0.0), blackInterp * blackTarget));
}
if (abs(adjustments.contrast) > epsilon) {
half mappedContrast = half(adjustments.contrast) * 0.3 + 1.0;
result.rgb = clamp(((result.rgb - half3(0.5)) * mappedContrast + half3(0.5)), 0.0, 1.0);
}
if (abs(adjustments.fade) > epsilon) {
result.rgb = fade(result.rgb, adjustments.fade);
}
if (abs(adjustments.saturation) > epsilon) {
float mappedSaturation = adjustments.saturation;
if (mappedSaturation > 0.0) {
mappedSaturation *= 1.05;
}
mappedSaturation += 1.0;
half satLuminance = dot(result.rgb, half3(0.2126, 0.7152, 0.0722));
half3 greyScaleColor = half3(satLuminance);
result.rgb = clamp(mix(greyScaleColor, result.rgb, mappedSaturation), 0.0, 1.0);
}
if (abs(adjustments.shadowsTintIntensity) > epsilon) {
result.rgb = tintShadows(result.rgb, adjustments.shadowsTintColor, adjustments.shadowsTintIntensity * 2.0);
}
if (abs(adjustments.highlightsTintIntensity) > epsilon) {
result.rgb = tintHighlights(result.rgb, adjustments.highlightsTintColor, adjustments.highlightsTintIntensity * 2.0);
}
if (abs(adjustments.exposure) > epsilon) {
float mag = adjustments.exposure * 1.045;
float power = 1.0 + abs(mag);
if (mag < 0.0) {
power = 1.0 / power;
}
result.r = 1.0 - pow((1.0 - result.r), power);
result.g = 1.0 - pow((1.0 - result.g), power);
result.b = 1.0 - pow((1.0 - result.b), power);
}
if (abs(adjustments.warmth) > epsilon) {
half3 yuvVector;
if (adjustments.warmth > 0.0) {
yuvVector = half3(0.1765, -0.1255, 0.0902);
} else {
yuvVector = -half3(0.0588, 0.1569, -0.1255);
}
half3 yuvColor = rgbToYuv(result.rgb);
half luma = yuvColor.r;
half curveScale = sin(luma * 3.14159);
yuvColor += 0.375 * adjustments.warmth * curveScale * yuvVector;
result.rgb = yuvToRgb(yuvColor);
}
if (abs(adjustments.vignette) > epsilon) {
const float midpoint = 0.7;
const float fuzziness = 0.62;
float radDist = length(in.texCoord - 0.5) / sqrt(0.5);
float mag = easeInOutSigmoid(radDist * midpoint, fuzziness) * adjustments.vignette * 0.645;
result.rgb = half3(mix(pow(float3(result.rgb), float3(1.0 / (1.0 - mag))), float3(0.0), mag * mag));
}
if (abs(adjustments.grain) > epsilon) {
const float grainSize = 2.3;
float3 rotOffset = float3(1.425, 3.892, 5.835);
float2 rotCoordsR = coordRot(in.texCoord, rotOffset.x);
half3 noise = half3(pnoise3D(float3(rotCoordsR * float2(adjustments.dimensions.x / grainSize, adjustments.dimensions.y / grainSize), 0.0)));
half3 lumcoeff = half3(0.299, 0.587, 0.114);
float luminance = dot(result.rgb, lumcoeff);
float lum = smoothstep(0.2, 0.0, luminance);
lum += luminance;
noise = mix(noise, half3(0.0), pow(lum, 4.0));
result.rgb = result.rgb + noise * adjustments.grain * 0.04;
}
return half4(result.rgb * result.a, result.a);
}
@@ -0,0 +1,72 @@
#include <metal_stdlib>
#include "EditorCommon.h"
#include "EditorUtils.h"
using namespace metal;
typedef struct {
float2 dimensions;
float2 position;
float aspectRatio;
float size;
float falloff;
float rotation;
} MediaEditorBlur;
fragment half4 blurRadialFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> sourceTexture [[texture(0)]],
texture2d<half, access::sample> blurTexture [[texture(1)]],
constant MediaEditorBlur& values [[ buffer(0) ]]
)
{
constexpr sampler sourceSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
constexpr sampler blurSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
half4 sourceColor = sourceTexture.sample(sourceSampler, in.texCoord);
half4 blurredColor = blurTexture.sample(blurSampler, in.texCoord);
float2 texCoord = float2(in.texCoord.x, (in.texCoord.y * values.aspectRatio + 0.5 - 0.5 * values.aspectRatio));
half distanceFromCenter = distance(values.position, texCoord);
half3 result = mix(blurredColor.rgb, sourceColor.rgb, smoothstep(1.0, values.falloff, clamp(distanceFromCenter / values.size, 0.0, 1.0)));
return half4(result, sourceColor.a);
}
fragment half4 blurLinearFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> sourceTexture [[texture(0)]],
texture2d<half, access::sample> blurTexture [[texture(1)]],
constant MediaEditorBlur& values [[ buffer(0) ]]
)
{
constexpr sampler sourceSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
constexpr sampler blurSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
half4 sourceColor = sourceTexture.sample(sourceSampler, in.texCoord);
half4 blurredColor = blurTexture.sample(blurSampler, in.texCoord);
float2 texCoord = float2(in.texCoord.x, (in.texCoord.y * values.aspectRatio + 0.5 - 0.5 * values.aspectRatio));
half distanceFromCenter = abs((texCoord.x - values.position.x) * sin(-values.rotation) + (texCoord.y - values.position.y) * cos(-values.rotation));
half3 result = mix(blurredColor.rgb, sourceColor.rgb, smoothstep(1.0, values.falloff, clamp(distanceFromCenter / values.size, 0.0, 1.0)));
return half4(result, sourceColor.a);
}
fragment half4 blurPortraitFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> sourceTexture [[texture(0)]],
texture2d<half, access::sample> blurTexture [[texture(1)]],
texture2d<half, access::sample> maskTexture [[texture(2)]],
constant MediaEditorBlur& values [[ buffer(0) ]]
)
{
constexpr sampler sourceSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
constexpr sampler blurSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
constexpr sampler maskSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
half4 sourceColor = sourceTexture.sample(sourceSampler, in.texCoord);
half4 blurredColor = blurTexture.sample(blurSampler, in.texCoord);
half4 maskColor = maskTexture.sample(maskSampler, in.texCoord);
half3 result = mix(blurredColor.rgb, sourceColor.rgb, maskColor.r);
return half4(result, sourceColor.a);
}
@@ -0,0 +1,9 @@
#include <metal_stdlib>
#pragma once
typedef struct {
float4 pos [[position]];
float2 texCoord;
float2 localPos;
} RasterizerData;
@@ -0,0 +1,50 @@
#include <metal_stdlib>
#include "EditorCommon.h"
using namespace metal;
typedef struct {
float4 pos;
float2 texCoord;
float2 localPos;
} VertexData;
vertex RasterizerData defaultVertexShader(uint vertexID [[vertex_id]],
constant VertexData *vertices [[buffer(0)]]) {
RasterizerData out;
out.pos = vector_float4(0.0, 0.0, 0.0, 1.0);
out.pos.xy = vertices[vertexID].pos.xy;
out.localPos = vertices[vertexID].localPos.xy;
out.texCoord = vertices[vertexID].texCoord;
return out;
}
fragment half4 defaultFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> texture [[texture(0)]]) {
constexpr sampler samplr(filter::linear, mag_filter::linear, min_filter::linear);
half4 color = texture.sample(samplr, in.texCoord);
return color;
}
fragment half histogramPrepareFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> texture [[texture(0)]]) {
constexpr sampler samplr(filter::linear, mag_filter::linear, min_filter::linear);
half3 color = texture.sample(samplr, in.texCoord).rgb;
half luma = color.r * 0.3 + color.g * 0.59 + color.b * 0.11;
return luma;
}
typedef struct {
float4 topColor;
float4 bottomColor;
} GradientColors;
fragment half4 gradientFragmentShader(RasterizerData in [[stage_in]],
constant GradientColors& colors [[buffer(0)]]) {
return half4(half3(mix(colors.topColor.rgb, colors.bottomColor.rgb, in.texCoord.y)), 1.0);
}
@@ -0,0 +1,46 @@
#include <metal_stdlib>
#include "EditorCommon.h"
#include "EditorUtils.h"
using namespace metal;
typedef struct {
float2 dimensions;
float roundness;
float alpha;
float isOpaque;
float empty;
} VideoEncodeParameters;
typedef struct {
float4 pos;
float2 texCoord;
float4 localPos;
} VertexData;
fragment half4 dualFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> texture [[texture(0)]],
texture2d<half, access::sample> mask [[texture(1)]],
constant VideoEncodeParameters& adjustments [[buffer(0)]]
) {
float2 R = float2(adjustments.dimensions.x, adjustments.dimensions.y);
float2 uv = (in.localPos - float2(0.5, 0.5)) * 2.0;
if (R.x > R.y) {
uv.y = uv.y * R.y / R.x;
} else {
uv.x = uv.x * R.x / R.y;
}
float aspectRatio = R.x / R.y;
constexpr sampler samplr(filter::linear, mag_filter::linear, min_filter::linear);
half4 color = texture.sample(samplr, in.texCoord);
float colorAlpha = min(1.0, adjustments.isOpaque * color.a + mask.sample(samplr, in.texCoord).r);
float t = 1.0 / adjustments.dimensions.y;
float side = 1.0 * aspectRatio;
float distance = smoothstep(t, -t, sdfRoundedRectangle(uv, float2(0.0, 0.0), float2(side, mix(1.0, side, adjustments.roundness)), side * adjustments.roundness));
return mix(half4(color.rgb, 0.0), half4(color.rgb, colorAlpha * adjustments.alpha), distance);
}
@@ -0,0 +1,119 @@
#include <metal_stdlib>
#include "EditorCommon.h"
#include "EditorUtils.h"
using namespace metal;
typedef struct {
uint histogramBins;
uint clipLimit;
uint totalPixelCountPerTile;
uint numberOfLUTs;
} MediaEditorEnhanceLUTGeneratorParameters;
fragment half rgbToLightnessFragmentShader(RasterizerData in [[ stage_in ]],
texture2d<half, access::sample> sourceTexture [[ texture(0) ]],
sampler colorSampler [[ sampler(0) ]],
constant float2 & scale [[buffer(0)]])
{
half4 color = sourceTexture.sample(colorSampler, in.texCoord * scale);
half3 hsl = rgbToHsl(color.rgb);
return hsl.b;
}
kernel void enhanceGenerateLUT(texture2d<float, access::write> outTexture [[texture(0)]],
device uint * histogramBuffer [[buffer(0)]],
constant MediaEditorEnhanceLUTGeneratorParameters & parameters [[buffer(1)]],
uint gid [[thread_position_in_grid]])
{
if (gid >= parameters.numberOfLUTs) {
return;
}
device uint *l = histogramBuffer + gid * parameters.histogramBins;
const uint histSize = parameters.histogramBins;
uint clipped = 0;
for (uint i = 0; i < histSize; ++i) {
if(l[i] > parameters.clipLimit) {
clipped += (l[i] - parameters.clipLimit);
l[i] = parameters.clipLimit;
}
}
const uint redistBatch = clipped / histSize;
uint residual = clipped - redistBatch * histSize;
for (uint i = 0; i < histSize; ++i) {
l[i] += redistBatch;
}
if (residual != 0) {
const uint residualStep = max(histSize / residual, (uint)1);
for (uint i = 0; i < histSize && residual > 0; i += residualStep, residual--) {
l[i]++;
}
}
uint sum = 0;
const float lutScale = (histSize - 1) / float(parameters.totalPixelCountPerTile);
for (uint index = 0; index < histSize; ++index) {
sum += l[index];
outTexture.write(round(sum * lutScale) / 255.0, uint2(index, gid));
}
}
half enhanceLookup(texture2d<half, access::sample> lutTexture, sampler lutSamper, float index, float x) {
return lutTexture.sample(lutSamper, float2(x, (index + 0.5)/lutTexture.get_height())).r;
}
fragment half4 enhanceColorLookupFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> sourceTexture [[texture(0)]],
texture2d<half, access::sample> lutTexture [[texture(1)]],
constant float2 & tileGridSize [[ buffer(0) ]],
constant float & intensity [[ buffer(1) ]]
)
{
constexpr sampler colorSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
constexpr sampler lutSampler(min_filter::linear, mag_filter::linear, address::clamp_to_zero);
float2 sourceCoord = in.texCoord;
half4 color = sourceTexture.sample(colorSampler, sourceCoord);
half3 hslColor = rgbToHsl(color.rgb);
float txf = sourceCoord.x * tileGridSize.x - 0.5;
float tx1 = floor(txf);
float tx2 = tx1 + 1.0;
float xa_p = txf - tx1;
float xa1_p = 1.0 - xa_p;
tx1 = max(tx1, 0.0);
tx2 = min(tx2, tileGridSize.x - 1.0);
float tyf = sourceCoord.y * tileGridSize.y - 0.5;
float ty1 = floor(tyf);
float ty2 = ty1 + 1.0;
float ya = tyf - ty1;
float ya1 = 1.0 - ya;
ty1 = max(ty1, 0.0);
ty2 = min(ty2, tileGridSize.y - 1.0);
float srcVal = hslColor.b;
float x = (srcVal * 255.0 + 0.5) / lutTexture.get_width();
half lutPlane1_ind1 = enhanceLookup(lutTexture, lutSampler, ty1 * tileGridSize.x + tx1, x);
half lutPlane1_ind2 = enhanceLookup(lutTexture, lutSampler, ty1 * tileGridSize.x + tx2, x);
half lutPlane2_ind1 = enhanceLookup(lutTexture, lutSampler, ty2 * tileGridSize.x + tx1, x);
half lutPlane2_ind2 = enhanceLookup(lutTexture, lutSampler, ty2 * tileGridSize.x + tx2, x);
half res = (lutPlane1_ind1 * xa1_p + lutPlane1_ind2 * xa_p) * ya1 + (lutPlane2_ind1 * xa1_p + lutPlane2_ind2 * xa_p) * ya;
half3 r = half3(hslColor.r, min(1.0, hslColor.g * 1.25), min(1.0, res * 1.1));
half3 rgbResult = hslToRgb(r);
return half4(mix(color.rgb, rgbResult, half(intensity)), color.a);
}
@@ -0,0 +1,28 @@
#include <metal_stdlib>
#pragma once
half getLuma(half3 color);
half3 rgbToHsv(half3 c);
half3 hsvToRgb(half3 c);
half3 rgbToHsl(half3 color);
half hueToRgb(half f1, half f2, half hue);
half3 hslToRgb(half3 hsl);
half3 rgbToYuv(half3 inP);
half3 yuvToRgb(half3 inP);
half easeInOutSigmoid(half value, half strength);
half powerCurve(half inVal, half mag);
float pnoise3D(float3 p);
float2 coordRot(float2 tc, float angle);
float sdfRoundedRectangle(float2 uv, float2 position, float2 size, float radius);
@@ -0,0 +1,209 @@
#include <metal_stdlib>
#include "EditorUtils.h"
using namespace metal;
half getLuma(half3 color) {
return (0.299 * color.r) + (0.587 * color.g) + (0.114 * color.b);
}
half3 rgbToHsv(half3 c) {
half4 K = half4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0);
half4 p = c.g < c.b ? half4(c.bg, K.wz) : half4(c.gb, K.xy);
half4 q = c.r < p.x ? half4(p.xyw, c.r) : half4(c.r, p.yzx);
half d = q.x - min(q.w, q.y);
half e = 1.0e-10;
return half3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x);
}
half3 hsvToRgb(half3 c) {
half4 K = half4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0);
half3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);
return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
}
half3 rgbToHsl(half3 color) {
half3 hsl;
half fMin = min(min(color.r, color.g), color.b);
half fMax = max(max(color.r, color.g), color.b);
half delta = fMax - fMin;
hsl.z = (fMax + fMin) / 2.0;
if (delta == 0.0) {
hsl.x = 0.0;
hsl.y = 0.0;
} else {
if (hsl.z < 0.5) {
hsl.y = delta / (fMax + fMin);
} else {
hsl.y = delta / (2.0 - fMax - fMin);
}
half deltaR = (((fMax - color.r) / 6.0) + (delta / 2.0)) / delta;
half deltaG = (((fMax - color.g) / 6.0) + (delta / 2.0)) / delta;
half deltaB = (((fMax - color.b) / 6.0) + (delta / 2.0)) / delta;
if (color.r == fMax) {
hsl.x = deltaB - deltaG;
} else if (color.g == fMax) {
hsl.x = (1.0 / 3.0) + deltaR - deltaB;
} else if (color.b == fMax) {
hsl.x = (2.0 / 3.0) + deltaG - deltaR;
}
if (hsl.x < 0.0) {
hsl.x += 1.0;
} else if (hsl.x > 1.0) {
hsl.x -= 1.0;
}
}
return hsl;
}
half hueToRgb(half f1, half f2, half hue) {
if (hue < 0.0) {
hue += 1.0;
} else if (hue > 1.0) {
hue -= 1.0;
}
half res;
if ((6.0 * hue) < 1.0) {
res = f1 + (f2 - f1) * 6.0 * hue;
} else if ((2.0 * hue) < 1.0) {
res = f2;
} else if ((3.0 * hue) < 2.0) {
res = f1 + (f2 - f1) * ((2.0 / 3.0) - hue) * 6.0;
} else {
res = f1;
}
return res;
}
half3 hslToRgb(half3 hsl) {
half3 rgb;
if (hsl.y == 0.0) {
rgb = half3(hsl.z);
} else {
half f2;
if (hsl.z < 0.5) {
f2 = hsl.z * (1.0 + hsl.y);
} else {
f2 = (hsl.z + hsl.y) - (hsl.y * hsl.z);
}
half f1 = 2.0 * hsl.z - f2;
rgb.r = hueToRgb(f1, f2, hsl.x + (1.0 / 3.0));
rgb.g = hueToRgb(f1, f2, hsl.x);
rgb.b = hueToRgb(f1, f2, hsl.x - (1.0 / 3.0));
}
return rgb;
}
half3 rgbToYuv(half3 inP) {
half3 outP;
outP.r = getLuma(inP);
outP.g = (1.0 / 1.772) * (inP.b - outP.r);
outP.b = (1.0 / 1.402) * (inP.r - outP.r);
return outP;
}
half3 yuvToRgb(half3 inP) {
float y = inP.r;
float u = inP.g;
float v = inP.b;
half3 outP;
outP.r = 1.402 * v + y;
outP.g = (y - (0.299 * 1.402 / 0.587) * v - (0.114 * 1.772 / 0.587) * u);
outP.b = 1.772 * u + y;
return outP;
}
half easeInOutSigmoid(half value, half strength) {
float t = 1.0 / (1.0 - strength);
if (value > 0.5) {
return 1.0 - pow(2.0 - 2.0 * value, t) * 0.5;
} else {
return pow(2.0 * value, t) * 0.5;
}
}
half powerCurve(half inVal, half mag) {
half outVal;
float power = 1.0 + abs(mag);
if (mag > 0.0) {
power = 1.0 / power;
}
inVal = 1.0 - inVal;
outVal = pow((1.0 - inVal), power);
return outVal;
}
float4 rnm(float2 tc) {
float noise = sin(dot(tc, float2(12.9898, 78.233))) * 43758.5453;
float noiseR = fract(noise) * 2.0-1.0;
float noiseG = fract(noise * 1.2154) * 2.0-1.0;
float noiseB = fract(noise * 1.3453) * 2.0-1.0;
float noiseA = fract(noise * 1.3647) * 2.0-1.0;
return float4(noiseR,noiseG,noiseB,noiseA);
}
float fade(float t) {
return t*t*t*(t*(t*6.0-15.0)+10.0);
}
float pnoise3D(float3 p) {
const half permTexUnit = 1.0 / 256.0;
const half permTexUnitHalf = 0.5 / 256.0;
float3 pi = permTexUnit * floor(p) + permTexUnitHalf;
float3 pf = fract(p);
// Noise contributions from (x=0, y=0), z=0 and z=1
float perm00 = rnm(pi.xy).a ;
float3 grad000 = rnm(float2(perm00, pi.z)).rgb * 4.0 - 1.0;
float n000 = dot(grad000, pf);
float3 grad001 = rnm(float2(perm00, pi.z + permTexUnit)).rgb * 4.0 - 1.0;
float n001 = dot(grad001, pf - float3(0.0, 0.0, 1.0));
// Noise contributions from (x=0, y=1), z=0 and z=1
float perm01 = rnm(pi.xy + float2(0.0, permTexUnit)).a ;
float3 grad010 = rnm(float2(perm01, pi.z)).rgb * 4.0 - 1.0;
float n010 = dot(grad010, pf - float3(0.0, 1.0, 0.0));
float3 grad011 = rnm(float2(perm01, pi.z + permTexUnit)).rgb * 4.0 - 1.0;
float n011 = dot(grad011, pf - float3(0.0, 1.0, 1.0));
// Noise contributions from (x=1, y=0), z=0 and z=1
float perm10 = rnm(pi.xy + float2(permTexUnit, 0.0)).a ;
float3 grad100 = rnm(float2(perm10, pi.z)).rgb * 4.0 - 1.0;
float n100 = dot(grad100, pf - float3(1.0, 0.0, 0.0));
float3 grad101 = rnm(float2(perm10, pi.z + permTexUnit)).rgb * 4.0 - 1.0;
float n101 = dot(grad101, pf - float3(1.0, 0.0, 1.0));
// Noise contributions from (x=1, y=1), z=0 and z=1
float perm11 = rnm(pi.xy + float2(permTexUnit, permTexUnit)).a ;
float3 grad110 = rnm(float2(perm11, pi.z)).rgb * 4.0 - 1.0;
float n110 = dot(grad110, pf - float3(1.0, 1.0, 0.0));
float3 grad111 = rnm(float2(perm11, pi.z + permTexUnit)).rgb * 4.0 - 1.0;
float n111 = dot(grad111, pf - float3(1.0, 1.0, 1.0));
// Blend contributions along x
float4 n_x = mix(float4(n000, n001, n010, n011), float4(n100, n101, n110, n111), fade(pf.x));
// Blend contributions along y
float2 n_xy = mix(n_x.xy, n_x.zw, fade(pf.y));
// Blend contributions along z
float n_xyz = mix(n_xy.x, n_xy.y, fade(pf.z));
return n_xyz;
}
float2 coordRot(float2 tc, float angle) {
float rotX = ((tc.x * 2.0 - 1.0) * cos(angle)) - ((tc.y * 2.0 - 1.0) * sin(angle));
float rotY = ((tc.y * 2.0 - 1.0) * cos(angle)) + ((tc.x * 2.0 - 1.0) * sin(angle));
rotX = rotX * 0.5 + 0.5;
rotY = rotY * 0.5 + 0.5;
return float2(rotX, rotY);
}
float sdfRoundedRectangle(float2 uv, float2 position, float2 size, float radius) {
float2 q = abs(uv - position) - size + radius;
return length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - radius;
}
@@ -0,0 +1,62 @@
#include <metal_stdlib>
#include "EditorCommon.h"
#include "EditorUtils.h"
using namespace metal;
//static inline float sRGBnonLinearNormToLinear(float normV) {
// if (normV <= 0.04045f) {
// normV *= (1.0f / 12.92f);
// } else {
// const float a = 0.055f;
// const float gamma = 2.4f;
// normV = (normV + a) * (1.0f / (1.0f + a));
// normV = pow(normV, gamma);
// }
// return normV;
//}
//static inline float4 sRGBGammaDecode(const float4 rgba) {
// float4 result = rgba;
// result.r = sRGBnonLinearNormToLinear(rgba.r);
// result.g = sRGBnonLinearNormToLinear(rgba.g);
// result.b = sRGBnonLinearNormToLinear(rgba.b);
// return result;
//}
static inline float4 BT709Decode(const float Y, const float Cb, const float Cr) {
float Yn = Y;
float Cbn = (Cb - (128.0f/255.0f));
float Crn = (Cr - (128.0f/255.0f));
float3 YCbCr = float3(Yn, Cbn, Crn);
const float3x3 kColorConversion709 = float3x3(float3(1.0, 1.0, 1.0),
float3(0.0f, -0.18732, 1.8556),
float3(1.5748, -0.46812, 0.0));
float3 rgb = kColorConversion709 * YCbCr;
rgb = saturate(rgb);
return float4(rgb.r, rgb.g, rgb.b, 1.0f);
}
fragment float4 bt709ToRGBFragmentShader(RasterizerData in [[stage_in]],
texture2d<half, access::sample> inYTexture [[texture(0)]],
texture2d<half, access::sample> inUVTexture [[texture(1)]]
)
{
constexpr sampler textureSampler (mag_filter::nearest, min_filter::nearest);
float Y = float(inYTexture.sample(textureSampler, in.texCoord).r);
half2 uvSamples = inUVTexture.sample(textureSampler, in.texCoord).rg;
float Cb = float(uvSamples[0]);
float Cr = float(uvSamples[1]);
float4 pixel = BT709Decode(Y, Cb, Cr);
//pixel = sRGBGammaDecode(pixel);
return pixel;
}
@@ -0,0 +1,160 @@
import Foundation
import Metal
import simd
struct MediaEditorAdjustments {
var dimensions: simd_float2
var aspectRatio: simd_float1
var shadows: simd_float1
var highlights: simd_float1
var contrast: simd_float1
var fade: simd_float1
var saturation: simd_float1
var shadowsTintIntensity: simd_float1
var shadowsTintColor: simd_float3
var highlightsTintIntensity: simd_float1
var highlightsTintColor: simd_float3
var exposure: simd_float1
var warmth: simd_float1
var grain: simd_float1
var vignette: simd_float1
var hasCurves: simd_float1
var empty: simd_float2
var hasValues: Bool {
let epsilon: simd_float1 = 0.005
if abs(self.shadows) > epsilon {
return true
}
if abs(self.highlights) > epsilon {
return true
}
if abs(self.contrast) > epsilon {
return true
}
if abs(self.fade) > epsilon {
return true
}
if abs(self.saturation) > epsilon {
return true
}
if abs(self.shadowsTintIntensity) > epsilon {
return true
}
if abs(self.highlightsTintIntensity) > epsilon {
return true
}
if abs(self.exposure) > epsilon {
return true
}
if abs(self.warmth) > epsilon {
return true
}
if abs(self.grain) > epsilon {
return true
}
if abs(self.vignette) > epsilon {
return true
}
if abs(self.hasCurves) > epsilon {
return true
}
return false
}
}
final class AdjustmentsRenderPass: DefaultRenderPass {
fileprivate var cachedTexture: MTLTexture?
var adjustments = MediaEditorAdjustments(
dimensions: simd_float2(1.0, 1.0),
aspectRatio: 0.0,
shadows: 0.0,
highlights: 0.0,
contrast: 0.0,
fade: 0.0,
saturation: 0.0,
shadowsTintIntensity: 0.0,
shadowsTintColor: simd_float3(0.0, 0.0, 0.0),
highlightsTintIntensity: 0.0,
highlightsTintColor: simd_float3(0.0, 0.0, 0.0),
exposure: 0.0,
warmth: 0.0,
grain: 0.0,
vignette: 0.0,
hasCurves: 0.0,
empty: simd_float2(0.0, 0.0)
)
var allCurve: [Float] = Array(repeating: 0, count: 200)
var redCurve: [Float] = Array(repeating: 0, count: 200)
var greenCurve: [Float] = Array(repeating: 0, count: 200)
var blueCurve: [Float] = Array(repeating: 0, count: 200)
override var fragmentShaderFunctionName: String {
return "adjustmentsFragmentShader"
}
override func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
// guard self.adjustments.hasValues else {
// return input
// }
self.setupVerticesBuffer(device: device)
let width = input.width
let height = input.height
if self.cachedTexture == nil || self.cachedTexture?.width != width || self.cachedTexture?.height != height {
self.adjustments.dimensions = simd_float2(Float(width), Float(height))
self.adjustments.aspectRatio = Float(width) / Float(height)
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = width
textureDescriptor.height = height
textureDescriptor.pixelFormat = input.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return input
}
self.cachedTexture = texture
texture.label = "adjustmentsTexture"
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 1.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(width), height: Double(height),
znear: -1.0, zfar: 1.0)
)
renderCommandEncoder.setFragmentTexture(input, index: 0)
renderCommandEncoder.setFragmentBytes(&self.adjustments, length: MemoryLayout<MediaEditorAdjustments>.size, index: 0)
let allCurve = self.allCurve
let redCurve = self.redCurve
let greenCurve = self.greenCurve
let blueCurve = self.blueCurve
renderCommandEncoder.setFragmentBytes(allCurve, length: MemoryLayout<Float>.size * 200, index: 1)
renderCommandEncoder.setFragmentBytes(redCurve, length: MemoryLayout<Float>.size * 200, index: 2)
renderCommandEncoder.setFragmentBytes(greenCurve, length: MemoryLayout<Float>.size * 200, index: 3)
renderCommandEncoder.setFragmentBytes(blueCurve, length: MemoryLayout<Float>.size * 200, index: 4)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return self.cachedTexture!
}
}
@@ -0,0 +1,252 @@
import Foundation
import Metal
import MetalPerformanceShaders
import simd
enum MediaEditorBlurMode {
case off
case radial
case linear
case portrait
}
struct MediaEditorBlur {
var dimensions: simd_float2
var position: simd_float2
var aspectRatio: simd_float1
var size: simd_float1
var falloff: simd_float1
var rotation: simd_float1
}
private final class BlurGaussianPass: RenderPass {
private var cachedTexture: MTLTexture?
fileprivate var blur: MPSImageGaussianBlur?
var updated: ((Data) -> Void)?
func setup(device: MTLDevice, library: MTLLibrary) {
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
return nil
}
func process(input: MTLTexture, intensity: Float, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
let radius = round(4.0 + intensity * 26.0)
if self.blur?.sigma != radius {
self.blur = MPSImageGaussianBlur(device: device, sigma: radius)
self.blur?.edgeMode = .clamp
}
if self.cachedTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = input.width
textureDescriptor.height = input.height
textureDescriptor.pixelFormat = input.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return input
}
self.cachedTexture = texture
}
if let blur = self.blur, let destinationTexture = self.cachedTexture {
blur.encode(commandBuffer: commandBuffer, sourceTexture: input, destinationTexture: destinationTexture)
}
return self.cachedTexture
}
}
private final class BlurLinearPass: DefaultRenderPass {
override var fragmentShaderFunctionName: String {
return "blurLinearFragmentShader"
}
func process(input: MTLTexture, blurredTexture: MTLTexture, values: MediaEditorBlur, output: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device)
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = output
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(input.width), height: Double(input.height),
znear: -1.0, zfar: 1.0)
)
var values = values
renderCommandEncoder.setFragmentTexture(input, index: 0)
renderCommandEncoder.setFragmentTexture(blurredTexture, index: 1)
renderCommandEncoder.setFragmentBytes(&values, length: MemoryLayout<MediaEditorBlur>.size, index: 0)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return output
}
}
private final class BlurRadialPass: DefaultRenderPass {
override var fragmentShaderFunctionName: String {
return "blurRadialFragmentShader"
}
func process(input: MTLTexture, blurredTexture: MTLTexture, values: MediaEditorBlur, output: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device)
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = output
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(input.width), height: Double(input.height),
znear: -1.0, zfar: 1.0)
)
var values = values
renderCommandEncoder.setFragmentTexture(input, index: 0)
renderCommandEncoder.setFragmentTexture(blurredTexture, index: 1)
renderCommandEncoder.setFragmentBytes(&values, length: MemoryLayout<MediaEditorBlur>.size, index: 0)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return output
}
}
private final class BlurPortraitPass: DefaultRenderPass {
fileprivate var cachedTexture: MTLTexture?
override var fragmentShaderFunctionName: String {
return "blurPortraitFragmentShader"
}
func process(input: MTLTexture, blurredTexture: MTLTexture, maskTexture: MTLTexture, values: MediaEditorBlur, output: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device)
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = output
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(input.width), height: Double(input.height),
znear: -1.0, zfar: 1.0)
)
var values = values
renderCommandEncoder.setFragmentTexture(input, index: 0)
renderCommandEncoder.setFragmentTexture(blurredTexture, index: 1)
renderCommandEncoder.setFragmentTexture(maskTexture, index: 2)
renderCommandEncoder.setFragmentBytes(&values, length: MemoryLayout<MediaEditorBlur>.size, index: 0)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return output
}
}
final class BlurRenderPass: RenderPass {
fileprivate var cachedTexture: MTLTexture?
var maskTexture: MTLTexture?
private let blurPass = BlurGaussianPass()
private let linearPass = BlurLinearPass()
private let radialPass = BlurRadialPass()
private let portraitPass = BlurPortraitPass()
var value = MediaEditorBlur(
dimensions: simd_float2(0.0, 0.0),
position: simd_float2(0.5, 0.5),
aspectRatio: 1.0,
size: 0.2,
falloff: 0.2,
rotation: 0.0
)
var intensity: simd_float1 = 0.0
var mode: MediaEditorBlurMode = .off
func setup(device: MTLDevice, library: MTLLibrary) {
self.blurPass.setup(device: device, library: library)
self.linearPass.setup(device: device, library: library)
self.radialPass.setup(device: device, library: library)
self.portraitPass.setup(device: device, library: library)
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.process(input: input, maskTexture: self.maskTexture, device: device, commandBuffer: commandBuffer)
}
func process(input: MTLTexture, maskTexture: MTLTexture?, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
guard self.intensity > 0.005 && self.mode != .off else {
return input
}
let width = input.width
let height = input.height
if self.cachedTexture == nil {
self.value.aspectRatio = Float(height) / Float(width)
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = width
textureDescriptor.height = height
textureDescriptor.pixelFormat = input.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return input
}
self.cachedTexture = texture
}
guard let blurredTexture = self.blurPass.process(input: input, intensity: self.intensity, device: device, commandBuffer: commandBuffer), let output = self.cachedTexture else {
return input
}
switch self.mode {
case .linear:
return self.linearPass.process(input: input, blurredTexture: blurredTexture, values: self.value, output: output, device: device, commandBuffer: commandBuffer)
case .radial:
return self.radialPass.process(input: input, blurredTexture: blurredTexture, values: self.value, output: output, device: device, commandBuffer: commandBuffer)
case .portrait:
if let maskTexture {
return self.portraitPass.process(input: input, blurredTexture: blurredTexture, maskTexture: maskTexture, values: self.value, output: output, device: device, commandBuffer: commandBuffer)
} else {
return input
}
default:
return input
}
}
}
@@ -0,0 +1,289 @@
import Foundation
import TelegramCore
import UrlEscaping
public func decodeCodableDrawingEntities(data: Data) -> [CodableDrawingEntity] {
if let codableEntities = try? JSONDecoder().decode([CodableDrawingEntity].self, from: data) {
return codableEntities
}
return []
}
public func decodeDrawingEntities(data: Data) -> [DrawingEntity] {
return decodeCodableDrawingEntities(data: data).map { $0.entity }
}
public enum CodableDrawingEntity: Equatable {
public static func == (lhs: CodableDrawingEntity, rhs: CodableDrawingEntity) -> Bool {
return lhs.entity.isEqual(to: rhs.entity)
}
case sticker(DrawingStickerEntity)
case text(DrawingTextEntity)
case simpleShape(DrawingSimpleShapeEntity)
case bubble(DrawingBubbleEntity)
case vector(DrawingVectorEntity)
case location(DrawingLocationEntity)
case link(DrawingLinkEntity)
case weather(DrawingWeatherEntity)
public init?(entity: DrawingEntity) {
if let entity = entity as? DrawingStickerEntity {
self = .sticker(entity)
} else if let entity = entity as? DrawingTextEntity {
self = .text(entity)
} else if let entity = entity as? DrawingSimpleShapeEntity {
self = .simpleShape(entity)
} else if let entity = entity as? DrawingBubbleEntity {
self = .bubble(entity)
} else if let entity = entity as? DrawingVectorEntity {
self = .vector(entity)
} else if let entity = entity as? DrawingLocationEntity {
self = .location(entity)
} else if let entity = entity as? DrawingLinkEntity {
self = .link(entity)
} else if let entity = entity as? DrawingWeatherEntity {
self = .weather(entity)
} else {
return nil
}
}
public var entity: DrawingEntity {
switch self {
case let .sticker(entity):
return entity
case let .text(entity):
return entity
case let .simpleShape(entity):
return entity
case let .bubble(entity):
return entity
case let .vector(entity):
return entity
case let .location(entity):
return entity
case let .link(entity):
return entity
case let .weather(entity):
return entity
}
}
private var coordinates: MediaArea.Coordinates? {
var position: CGPoint?
var size: CGSize?
var rotation: CGFloat?
var scale: CGFloat?
var cornerRadius: Double?
switch self {
case let .location(entity):
position = entity.position
size = entity.renderImage?.size
rotation = entity.rotation
scale = entity.scale
if let size {
cornerRadius = 10.0 / (size.width * entity.scale)
}
case let .sticker(entity):
var entityPosition = entity.position
var entitySize = entity.baseSize
let entityRotation = entity.rotation
let entityScale = entity.scale
if case .message = entity.content {
let offset: CGFloat = 16.18 * entityScale
entitySize = CGSize(width: entitySize.width - 38.0, height: entitySize.height - 4.0)
entityPosition = CGPoint(x: entityPosition.x + offset * cos(entityRotation), y: entityPosition.y + offset * sin(entityRotation))
}
position = entityPosition
size = entitySize
rotation = entityRotation
scale = entityScale
case let .link(entity):
position = entity.position
rotation = entity.rotation
scale = entity.scale
if let entitySize = entity.renderImage?.size {
if entity.whiteImage != nil {
cornerRadius = 38.0 / (entitySize.width * entity.scale)
size = CGSize(width: entitySize.width - 28.0, height: entitySize.height - 26.0)
} else {
cornerRadius = 10.0 / (entitySize.width * entity.scale)
size = entitySize
}
}
case let .weather(entity):
position = entity.position
size = entity.renderImage?.size
rotation = entity.rotation
scale = entity.scale
if let size {
cornerRadius = (size.height * 0.17) / size.width
}
default:
return nil
}
guard let position, let size, let scale, let rotation else {
return nil
}
let width = size.width * scale / 1080.0 * 100.0
let height = size.height * scale / 1920.0 * 100.0
return MediaArea.Coordinates(
x: position.x / 1080.0 * 100.0,
y: position.y / 1920.0 * 100.0,
width: width,
height: height,
rotation: rotation / .pi * 180.0,
cornerRadius: cornerRadius.flatMap { $0 * 100.0 }
)
}
public var mediaArea: MediaArea? {
guard let coordinates = self.coordinates else {
return nil
}
switch self {
case let .location(entity):
return .venue(
coordinates: coordinates,
venue: MediaArea.Venue(
latitude: entity.location.latitude,
longitude: entity.location.longitude,
venue: entity.location.venue,
address: entity.location.address,
queryId: entity.queryId,
resultId: entity.resultId
)
)
case let .sticker(entity):
if case let .file(_, type) = entity.content, case let .reaction(reaction, style) = type {
var flags: MediaArea.ReactionFlags = []
if case .black = style {
flags.insert(.isDark)
}
if entity.mirrored {
flags.insert(.isFlipped)
}
return .reaction(
coordinates: coordinates,
reaction: reaction,
flags: flags
)
} else if case let .message(messageIds, _, _, _, _) = entity.content, let messageId = messageIds.first {
return .channelMessage(
coordinates: coordinates,
messageId: messageId
)
} else if case let .gift(gift, _) = entity.content {
return .starGift(
coordinates: coordinates,
slug: gift.slug
)
} else {
return nil
}
case let .link(entity):
return .link(
coordinates: coordinates,
url: explicitUrl(entity.url)
)
case let .weather(entity):
let color: UInt32
switch entity.style {
case .white:
color = 0xffffffff
case .black:
color = 0xff000000
case .transparent:
color = 0x51000000
case .custom:
color = entity.color.toUIColor().argb
}
return .weather(
coordinates: coordinates,
emoji: entity.emoji,
temperature: entity.temperature,
color: Int32(bitPattern: color)
)
default:
return nil
}
}
}
extension CodableDrawingEntity: Codable {
private enum CodingKeys: String, CodingKey {
case type
case entity
}
private enum EntityType: Int, Codable {
case sticker
case text
case simpleShape
case bubble
case vector
case location
case link
case weather
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
let type = try container.decode(EntityType.self, forKey: .type)
switch type {
case .sticker:
self = .sticker(try container.decode(DrawingStickerEntity.self, forKey: .entity))
case .text:
self = .text(try container.decode(DrawingTextEntity.self, forKey: .entity))
case .simpleShape:
self = .simpleShape(try container.decode(DrawingSimpleShapeEntity.self, forKey: .entity))
case .bubble:
self = .bubble(try container.decode(DrawingBubbleEntity.self, forKey: .entity))
case .vector:
self = .vector(try container.decode(DrawingVectorEntity.self, forKey: .entity))
case .location:
self = .location(try container.decode(DrawingLocationEntity.self, forKey: .entity))
case .link:
self = .link(try container.decode(DrawingLinkEntity.self, forKey: .entity))
case .weather:
self = .weather(try container.decode(DrawingWeatherEntity.self, forKey: .entity))
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
switch self {
case let .sticker(payload):
try container.encode(EntityType.sticker, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .text(payload):
try container.encode(EntityType.text, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .simpleShape(payload):
try container.encode(EntityType.simpleShape, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .bubble(payload):
try container.encode(EntityType.bubble, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .vector(payload):
try container.encode(EntityType.vector, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .location(payload):
try container.encode(EntityType.location, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .link(payload):
try container.encode(EntityType.link, forKey: .type)
try container.encode(payload, forKey: .entity)
case let .weather(payload):
try container.encode(EntityType.weather, forKey: .type)
try container.encode(payload, forKey: .entity)
}
}
}
@@ -0,0 +1,144 @@
import Foundation
import UIKit
import Display
import AccountContext
public final class DrawingBubbleEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case drawType
case color
case lineWidth
case referenceDrawingSize
case position
case size
case rotation
case tailPosition
case renderImage
}
public enum DrawType: Codable {
case fill
case stroke
}
public var uuid: UUID
public let isAnimated: Bool
public var drawType: DrawType
public var color: DrawingColor
public var lineWidth: CGFloat
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var size: CGSize
public var rotation: CGFloat
public var tailPosition: CGPoint
public var center: CGPoint {
return self.position
}
public var scale: CGFloat = 1.0
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public init(drawType: DrawType, color: DrawingColor, lineWidth: CGFloat) {
self.uuid = UUID()
self.isAnimated = false
self.drawType = drawType
self.color = color
self.lineWidth = lineWidth
self.referenceDrawingSize = .zero
self.position = .zero
self.size = CGSize(width: 1.0, height: 1.0)
self.rotation = 0.0
self.tailPosition = CGPoint(x: 0.16, y: 0.18)
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.isAnimated = false
self.drawType = try container.decode(DrawType.self, forKey: .drawType)
self.color = try container.decode(DrawingColor.self, forKey: .color)
self.lineWidth = try container.decode(CGFloat.self, forKey: .lineWidth)
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.size = try container.decode(CGSize.self, forKey: .size)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
self.tailPosition = try container.decode(CGPoint.self, forKey: .tailPosition)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.drawType, forKey: .drawType)
try container.encode(self.color, forKey: .color)
try container.encode(self.lineWidth, forKey: .lineWidth)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.size, forKey: .size)
try container.encode(self.rotation, forKey: .rotation)
try container.encode(self.tailPosition, forKey: .tailPosition)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingBubbleEntity(drawType: self.drawType, color: self.color, lineWidth: self.lineWidth)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.size = self.size
newEntity.rotation = self.rotation
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingBubbleEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.drawType != other.drawType {
return false
}
if self.color != other.color {
return false
}
if self.lineWidth != other.lineWidth {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.size != other.size {
return false
}
if self.rotation != other.rotation {
return false
}
if self.tailPosition != other.tailPosition {
return false
}
return true
}
}
@@ -0,0 +1,161 @@
import Foundation
import UIKit
import simd
public struct DrawingColor: Equatable, Codable {
private enum CodingKeys: String, CodingKey {
case red
case green
case blue
case alpha
case position
}
public static var clear = DrawingColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
public var red: CGFloat
public var green: CGFloat
public var blue: CGFloat
public var alpha: CGFloat
public var position: CGPoint?
public var isClear: Bool {
return self.red.isZero && self.green.isZero && self.blue.isZero && self.alpha.isZero
}
public init(
red: CGFloat,
green: CGFloat,
blue: CGFloat,
alpha: CGFloat = 1.0,
position: CGPoint? = nil
) {
self.red = red
self.green = green
self.blue = blue
self.alpha = alpha
self.position = position
}
public init(color: UIColor) {
var red: CGFloat = 0.0
var green: CGFloat = 0.0
var blue: CGFloat = 0.0
var alpha: CGFloat = 1.0
if color.getRed(&red, green: &green, blue: &blue, alpha: &alpha) {
self.init(red: red, green: green, blue: blue, alpha: alpha)
} else if color.getWhite(&red, alpha: &alpha) {
self.init(red: red, green: red, blue: red, alpha: alpha)
} else {
self.init(red: 0.0, green: 0.0, blue: 0.0)
}
}
public init(rgb: UInt32) {
self.init(color: UIColor(rgb: rgb))
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.red = try container.decode(CGFloat.self, forKey: .red)
self.green = try container.decode(CGFloat.self, forKey: .green)
self.blue = try container.decode(CGFloat.self, forKey: .blue)
self.alpha = try container.decode(CGFloat.self, forKey: .alpha)
self.position = try container.decodeIfPresent(CGPoint.self, forKey: .position)
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.red, forKey: .red)
try container.encode(self.green, forKey: .green)
try container.encode(self.blue, forKey: .blue)
try container.encode(self.alpha, forKey: .alpha)
try container.encodeIfPresent(self.position, forKey: .position)
}
public func withUpdatedRed(_ red: CGFloat) -> DrawingColor {
return DrawingColor(
red: red,
green: self.green,
blue: self.blue,
alpha: self.alpha
)
}
public func withUpdatedGreen(_ green: CGFloat) -> DrawingColor {
return DrawingColor(
red: self.red,
green: green,
blue: self.blue,
alpha: self.alpha
)
}
public func withUpdatedBlue(_ blue: CGFloat) -> DrawingColor {
return DrawingColor(
red: self.red,
green: self.green,
blue: blue,
alpha: self.alpha
)
}
public func withUpdatedAlpha(_ alpha: CGFloat) -> DrawingColor {
return DrawingColor(
red: self.red,
green: self.green,
blue: self.blue,
alpha: alpha,
position: self.position
)
}
public func withUpdatedPosition(_ position: CGPoint) -> DrawingColor {
return DrawingColor(
red: self.red,
green: self.green,
blue: self.blue,
alpha: self.alpha,
position: position
)
}
public func toUIColor() -> UIColor {
return UIColor(
red: self.red,
green: self.green,
blue: self.blue,
alpha: self.alpha
)
}
public func toCGColor() -> CGColor {
return self.toUIColor().cgColor
}
public func toFloat4() -> vector_float4 {
return [
simd_float1(self.red),
simd_float1(self.green),
simd_float1(self.blue),
simd_float1(self.alpha)
]
}
public static func ==(lhs: DrawingColor, rhs: DrawingColor) -> Bool {
if lhs.red != rhs.red {
return false
}
if lhs.green != rhs.green {
return false
}
if lhs.blue != rhs.blue {
return false
}
if lhs.alpha != rhs.alpha {
return false
}
return true
}
}
@@ -0,0 +1,22 @@
import Foundation
import UIKit
public protocol DrawingEntity: AnyObject {
var uuid: UUID { get set }
var isAnimated: Bool { get }
var center: CGPoint { get }
var isMedia: Bool { get }
var lineWidth: CGFloat { get set }
var color: DrawingColor { get set }
var scale: CGFloat { get set }
func duplicate(copy: Bool) -> DrawingEntity
var renderImage: UIImage? { get set }
var renderSubEntities: [DrawingEntity]? { get set }
func isEqual(to other: DrawingEntity) -> Bool
}
@@ -0,0 +1,260 @@
import Foundation
import UIKit
import Display
import AccountContext
import TextFormat
import Postbox
import TelegramCore
public final class DrawingLinkEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case url
case name
case webpage
case positionBelowText
case largeMedia
case expandedSize
case style
case color
case hasCustomColor
case referenceDrawingSize
case position
case width
case scale
case rotation
case renderImage
case whiteImage
case blackImage
}
public enum Style: Codable, Equatable {
case white
case black
case transparent
case custom
case blur
}
public var uuid: UUID
public var isAnimated: Bool {
return false
}
public var url: String
public var name: String
public var webpage: TelegramMediaWebpage?
public var positionBelowText: Bool
public var largeMedia: Bool?
public var expandedSize: CGSize?
public var style: Style
public var color: DrawingColor = DrawingColor(color: .white) {
didSet {
if self.color.toUIColor().argb == UIColor.white.argb {
self.style = .white
self.hasCustomColor = false
} else {
self.style = .custom
self.hasCustomColor = true
}
}
}
public var hasCustomColor = false
public var lineWidth: CGFloat = 0.0
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var width: CGFloat
public var scale: CGFloat {
didSet {
self.scale = min(2.5, self.scale)
}
}
public var rotation: CGFloat
public var center: CGPoint {
return self.position
}
public var whiteImage: UIImage?
public var blackImage: UIImage?
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public init(
url: String,
name: String,
webpage: TelegramMediaWebpage?,
positionBelowText: Bool,
largeMedia: Bool?,
style: Style
) {
self.uuid = UUID()
self.url = url
self.name = name
self.webpage = webpage
self.positionBelowText = positionBelowText
self.largeMedia = largeMedia
self.style = style
self.referenceDrawingSize = .zero
self.position = .zero
self.width = 100.0
self.scale = 1.0
self.rotation = 0.0
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.url = try container.decode(String.self, forKey: .url)
self.name = try container.decode(String.self, forKey: .name)
self.positionBelowText = try container.decode(Bool.self, forKey: .positionBelowText)
self.largeMedia = try container.decodeIfPresent(Bool.self, forKey: .largeMedia)
self.style = try container.decode(Style.self, forKey: .style)
if let webpageData = try container.decodeIfPresent(Data.self, forKey: .webpage) {
self.webpage = PostboxDecoder(buffer: MemoryBuffer(data: webpageData)).decodeRootObject() as? TelegramMediaWebpage
} else {
self.webpage = nil
}
self.color = try container.decodeIfPresent(DrawingColor.self, forKey: .color) ?? DrawingColor(color: .white)
self.hasCustomColor = try container.decodeIfPresent(Bool.self, forKey: .hasCustomColor) ?? false
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.width = try container.decode(CGFloat.self, forKey: .width)
self.scale = try container.decode(CGFloat.self, forKey: .scale)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
if let imagePath = try container.decodeIfPresent(String.self, forKey: .whiteImage), let image = UIImage(contentsOfFile: fullEntityMediaPath(imagePath)) {
self.whiteImage = image
}
if let imagePath = try container.decodeIfPresent(String.self, forKey: .blackImage), let image = UIImage(contentsOfFile: fullEntityMediaPath(imagePath)) {
self.blackImage = image
}
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.url, forKey: .url)
try container.encode(self.name, forKey: .name)
try container.encode(self.positionBelowText, forKey: .positionBelowText)
try container.encodeIfPresent(self.largeMedia, forKey: .largeMedia)
if let webpage = self.webpage {
let encoder = PostboxEncoder()
encoder.encodeRootObject(webpage)
let webpageData = encoder.makeData()
try container.encode(webpageData, forKey: .webpage)
} else {
try container.encodeNil(forKey: .webpage)
}
try container.encode(self.style, forKey: .style)
try container.encode(self.color, forKey: .color)
try container.encode(self.hasCustomColor, forKey: .hasCustomColor)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.width, forKey: .width)
try container.encode(self.scale, forKey: .scale)
try container.encode(self.rotation, forKey: .rotation)
if let image = self.whiteImage {
let imagePath = "\(self.uuid)_white.png"
let fullImagePath = fullEntityMediaPath(imagePath)
if let imageData = image.pngData() {
try? FileManager.default.createDirectory(atPath: entitiesPath(), withIntermediateDirectories: true)
try? imageData.write(to: URL(fileURLWithPath: fullImagePath))
try container.encodeIfPresent(imagePath, forKey: .whiteImage)
}
}
if let image = self.blackImage {
let imagePath = "\(self.uuid)black.png"
let fullImagePath = fullEntityMediaPath(imagePath)
if let imageData = image.pngData() {
try? FileManager.default.createDirectory(atPath: entitiesPath(), withIntermediateDirectories: true)
try? imageData.write(to: URL(fileURLWithPath: fullImagePath))
try container.encodeIfPresent(imagePath, forKey: .blackImage)
}
}
if let renderImage = self.renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingLinkEntity(url: self.url, name: self.name, webpage: self.webpage, positionBelowText: self.positionBelowText, largeMedia: self.largeMedia, style: self.style)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.width = self.width
newEntity.scale = self.scale
newEntity.rotation = self.rotation
newEntity.whiteImage = self.whiteImage
newEntity.blackImage = self.blackImage
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingLinkEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.url != other.url {
return false
}
if self.name != other.name {
return false
}
if self.webpage != other.webpage {
return false
}
if self.positionBelowText != other.positionBelowText {
return false
}
if self.largeMedia != other.largeMedia {
return false
}
if self.style != other.style {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.width != other.width {
return false
}
if self.scale != other.scale {
return false
}
if self.rotation != other.rotation {
return false
}
return true
}
}
@@ -0,0 +1,216 @@
import Foundation
import UIKit
import Display
import AccountContext
import TextFormat
import Postbox
import TelegramCore
public final class DrawingLocationEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case title
case style
case color
case hasCustomColor
case location
case icon
case queryId
case resultId
case referenceDrawingSize
case position
case width
case scale
case rotation
case renderImage
}
public enum Style: Codable, Equatable {
case white
case black
case transparent
case custom
case blur
}
public var uuid: UUID
public var isAnimated: Bool {
return false
}
public var title: String
public var style: Style
public var location: TelegramMediaMap
public var icon: TelegramMediaFile?
public var queryId: Int64?
public var resultId: String?
public var color: DrawingColor = DrawingColor(color: .white) {
didSet {
if self.color.toUIColor().argb == UIColor.white.argb {
self.style = .white
self.hasCustomColor = false
} else {
self.style = .custom
self.hasCustomColor = true
}
}
}
public var hasCustomColor = false
public var lineWidth: CGFloat = 0.0
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var width: CGFloat
public var scale: CGFloat {
didSet {
self.scale = min(2.5, self.scale)
}
}
public var rotation: CGFloat
public var center: CGPoint {
return self.position
}
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public init(title: String, style: Style, location: TelegramMediaMap, icon: TelegramMediaFile?, queryId: Int64?, resultId: String?) {
self.uuid = UUID()
self.title = title
self.style = style
self.location = location
self.icon = icon
self.queryId = queryId
self.resultId = resultId
self.referenceDrawingSize = .zero
self.position = .zero
self.width = 100.0
self.scale = 1.0
self.rotation = 0.0
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.title = try container.decode(String.self, forKey: .title)
self.style = try container.decode(Style.self, forKey: .style)
self.color = try container.decodeIfPresent(DrawingColor.self, forKey: .color) ?? DrawingColor(color: .white)
self.hasCustomColor = try container.decodeIfPresent(Bool.self, forKey: .hasCustomColor) ?? false
if let locationData = try container.decodeIfPresent(Data.self, forKey: .location) {
self.location = PostboxDecoder(buffer: MemoryBuffer(data: locationData)).decodeRootObject() as! TelegramMediaMap
} else {
fatalError()
}
if let iconData = try container.decodeIfPresent(Data.self, forKey: .icon) {
self.icon = PostboxDecoder(buffer: MemoryBuffer(data: iconData)).decodeRootObject() as? TelegramMediaFile
}
self.queryId = try container.decodeIfPresent(Int64.self, forKey: .queryId)
self.resultId = try container.decodeIfPresent(String.self, forKey: .resultId)
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.width = try container.decode(CGFloat.self, forKey: .width)
self.scale = try container.decode(CGFloat.self, forKey: .scale)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.title, forKey: .title)
try container.encode(self.style, forKey: .style)
try container.encode(self.color, forKey: .color)
try container.encode(self.hasCustomColor, forKey: .hasCustomColor)
var encoder = PostboxEncoder()
encoder.encodeRootObject(self.location)
let locationData = encoder.makeData()
try container.encode(locationData, forKey: .location)
if let icon = self.icon {
encoder = PostboxEncoder()
encoder.encodeRootObject(icon)
let iconData = encoder.makeData()
try container.encode(iconData, forKey: .icon)
}
try container.encodeIfPresent(self.queryId, forKey: .queryId)
try container.encodeIfPresent(self.resultId, forKey: .resultId)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.width, forKey: .width)
try container.encode(self.scale, forKey: .scale)
try container.encode(self.rotation, forKey: .rotation)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingLocationEntity(title: self.title, style: self.style, location: self.location, icon: self.icon, queryId: self.queryId, resultId: self.resultId)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.width = self.width
newEntity.scale = self.scale
newEntity.rotation = self.rotation
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingLocationEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.title != other.title {
return false
}
if self.style != other.style {
return false
}
if self.location != other.location {
return false
}
if self.queryId != other.queryId {
return false
}
if self.resultId != other.resultId {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.width != other.width {
return false
}
if self.scale != other.scale {
return false
}
if self.rotation != other.rotation {
return false
}
return true
}
}
@@ -0,0 +1,121 @@
import Foundation
import UIKit
import Display
import SwiftSignalKit
import TelegramCore
import AccountContext
import Photos
public final class DrawingMediaEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case size
case referenceDrawingSize
case position
case scale
case rotation
case mirrored
}
public var uuid: UUID
public let size: CGSize
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var scale: CGFloat
public var rotation: CGFloat
public var mirrored: Bool
public var color: DrawingColor = DrawingColor.clear
public var lineWidth: CGFloat = 0.0
public var center: CGPoint {
return self.position
}
public var baseSize: CGSize {
return self.size
}
public var isAnimated: Bool {
return false
}
public var isMedia: Bool {
return true
}
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public init(size: CGSize) {
self.uuid = UUID()
self.size = size
self.referenceDrawingSize = .zero
self.position = CGPoint()
self.scale = 1.0
self.rotation = 0.0
self.mirrored = false
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.size = try container.decode(CGSize.self, forKey: .size)
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.scale = try container.decode(CGFloat.self, forKey: .scale)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
self.mirrored = try container.decode(Bool.self, forKey: .mirrored)
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.size, forKey: .size)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.scale, forKey: .scale)
try container.encode(self.rotation, forKey: .rotation)
try container.encode(self.mirrored, forKey: .mirrored)
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingMediaEntity(size: self.size)
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.scale = self.scale
newEntity.rotation = self.rotation
newEntity.mirrored = self.mirrored
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingMediaEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.size != other.size {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.scale != other.scale {
return false
}
if self.rotation != other.rotation {
return false
}
if self.mirrored != other.mirrored {
return false
}
return true
}
}
@@ -0,0 +1,150 @@
import Foundation
import UIKit
import Display
import AccountContext
public final class DrawingSimpleShapeEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case shapeType
case drawType
case color
case lineWidth
case referenceDrawingSize
case position
case size
case rotation
case renderImage
}
public enum ShapeType: Codable {
case rectangle
case ellipse
case star
}
public enum DrawType: Codable {
case fill
case stroke
}
public var uuid: UUID
public let isAnimated: Bool
public var shapeType: ShapeType
public var drawType: DrawType
public var color: DrawingColor
public var lineWidth: CGFloat
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var size: CGSize
public var rotation: CGFloat
public var center: CGPoint {
return self.position
}
public var scale: CGFloat = 1.0
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public init(shapeType: ShapeType, drawType: DrawType, color: DrawingColor, lineWidth: CGFloat) {
self.uuid = UUID()
self.isAnimated = false
self.shapeType = shapeType
self.drawType = drawType
self.color = color
self.lineWidth = lineWidth
self.referenceDrawingSize = .zero
self.position = .zero
self.size = CGSize(width: 1.0, height: 1.0)
self.rotation = 0.0
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.isAnimated = false
self.shapeType = try container.decode(ShapeType.self, forKey: .shapeType)
self.drawType = try container.decode(DrawType.self, forKey: .drawType)
self.color = try container.decode(DrawingColor.self, forKey: .color)
self.lineWidth = try container.decode(CGFloat.self, forKey: .lineWidth)
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.size = try container.decode(CGSize.self, forKey: .size)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.shapeType, forKey: .shapeType)
try container.encode(self.drawType, forKey: .drawType)
try container.encode(self.color, forKey: .color)
try container.encode(self.lineWidth, forKey: .lineWidth)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.size, forKey: .size)
try container.encode(self.rotation, forKey: .rotation)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingSimpleShapeEntity(shapeType: self.shapeType, drawType: self.drawType, color: self.color, lineWidth: self.lineWidth)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.size = self.size
newEntity.rotation = self.rotation
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingSimpleShapeEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.shapeType != other.shapeType {
return false
}
if self.drawType != other.drawType {
return false
}
if self.color != other.color {
return false
}
if self.lineWidth != other.lineWidth {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.size != other.size {
return false
}
if self.rotation != other.rotation {
return false
}
return true
}
}
@@ -0,0 +1,491 @@
import Foundation
import UIKit
import Display
import AccountContext
import Postbox
import TelegramCore
func entitiesPath() -> String {
return NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0] + "/mediaEntities"
}
func fullEntityMediaPath(_ path: String) -> String {
return NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0] + "/mediaEntities/" + path
}
public final class DrawingStickerEntity: DrawingEntity, Codable {
public enum DecodingError: Error {
case generic
}
public enum Content: Equatable {
public enum ImageType: Equatable {
case sticker
case rectangle
case dualPhoto
}
public enum FileType: Equatable {
public enum ReactionStyle: Int32 {
case white
case black
}
case sticker
case reaction(MessageReaction.Reaction, ReactionStyle)
}
case file(FileMediaReference, FileType)
case image(UIImage, ImageType)
case animatedImage(Data, UIImage)
case video(TelegramMediaFile)
case dualVideoReference(Bool)
case message([MessageId], CGSize, TelegramMediaFile?, CGRect?, CGFloat?)
case gift(StarGift.UniqueGift, CGSize)
public static func == (lhs: Content, rhs: Content) -> Bool {
switch lhs {
case let .file(lhsFile, lhsFileType):
if case let .file(rhsFile, rhsFileType) = rhs {
return lhsFile.media.fileId == rhsFile.media.fileId && lhsFileType == rhsFileType
} else {
return false
}
case let .image(lhsImage, lhsImageType):
if case let .image(rhsImage, rhsImageType) = rhs {
return lhsImage === rhsImage && lhsImageType == rhsImageType
} else {
return false
}
case let .animatedImage(lhsData, lhsThumbnailImage):
if case let .animatedImage(rhsData, rhsThumbnailImage) = lhs {
return lhsData == rhsData && lhsThumbnailImage === rhsThumbnailImage
} else {
return false
}
case let .video(lhsFile):
if case let .video(rhsFile) = rhs {
return lhsFile.fileId == rhsFile.fileId
} else {
return false
}
case let .dualVideoReference(isAdditional):
if case .dualVideoReference(isAdditional) = rhs {
return true
} else {
return false
}
case let .message(lhsMessageIds, lhsSize, lhsFile, lhsMediaFrame, lhsCornerRadius):
if case let .message(rhsMessageIds, rhsSize, rhsFile, rhsMediaFrame, rhsCornerRadius) = rhs {
return lhsMessageIds == rhsMessageIds && lhsSize == rhsSize && lhsFile?.fileId == rhsFile?.fileId && lhsMediaFrame == rhsMediaFrame && lhsCornerRadius == rhsCornerRadius
} else {
return false
}
case let .gift(lhsGift, lhsSize):
if case let .gift(rhsGift, rhsSize) = rhs {
return lhsGift == rhsGift && lhsSize == rhsSize
} else {
return false
}
}
}
}
private enum CodingKeys: String, CodingKey {
case uuid
case file
case reaction
case reactionStyle
case imagePath
case animatedImagePath
case videoFile
case isRectangle
case isDualPhoto
case dualVideo
case isAdditionalVideo
case messageIds
case messageFile
case messageSize
case messageMediaRect
case messageMediaCornerRadius
case gift
case referenceDrawingSize
case position
case scale
case rotation
case mirrored
case isExplicitlyStatic
case canCutOut
case renderImage
case renderSubEntities
}
public var uuid: UUID
public var content: Content
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var scale: CGFloat {
didSet {
if case let .file(_, type) = self.content, case .reaction = type {
self.scale = max(0.59, min(1.77, self.scale))
} else if case .message = self.content {
self.scale = max(2.5, self.scale)
} else if case .gift = self.content {
self.scale = max(2.5, self.scale)
}
}
}
public var rotation: CGFloat
public var mirrored: Bool
public var canCutOut = false
public var isExplicitlyStatic: Bool
public var color: DrawingColor = DrawingColor.clear
public var lineWidth: CGFloat = 0.0
public var secondaryRenderImage: UIImage?
public var overlayRenderImage: UIImage?
public var tertiaryRenderImage: UIImage?
public var quaternaryRenderImage: UIImage?
public var center: CGPoint {
return self.position
}
public var baseSize: CGSize {
let size = max(10.0, min(self.referenceDrawingSize.width, self.referenceDrawingSize.height) * 0.25)
let dimensions: CGSize
switch self.content {
case let .image(image, _):
dimensions = image.size
case let .animatedImage(_, thumbnailImage):
dimensions = thumbnailImage.size
case let .file(file, type):
if case .reaction = type {
dimensions = CGSize(width: 512.0, height: 512.0)
} else {
dimensions = file.media.dimensions?.cgSize ?? CGSize(width: 512.0, height: 512.0)
}
case let .video(file):
dimensions = file.dimensions?.cgSize ?? CGSize(width: 512.0, height: 512.0)
case .dualVideoReference:
dimensions = CGSize(width: 512.0, height: 512.0)
case let .message(_, size, _, _, _):
dimensions = size
case let .gift(_, size):
dimensions = size
}
let boundingSize = CGSize(width: size, height: size)
return dimensions.fitted(boundingSize)
}
public var isAnimated: Bool {
switch self.content {
case let .file(file, type):
if self.isExplicitlyStatic {
return false
} else {
switch type {
case .reaction:
return false
default:
return file.media.isAnimatedSticker || file.media.isVideoSticker || file.media.mimeType == "video/webm"
}
}
case .image:
return false
case .animatedImage:
return true
case .video:
return true
case .dualVideoReference:
return true
case .message, .gift:
return !(self.renderSubEntities ?? []).isEmpty
}
}
public var isRectangle: Bool {
switch self.content {
case let .image(_, imageType):
return imageType == .rectangle
case .video:
return true
case .message, .gift:
return true
default:
return false
}
}
public var isMedia: Bool {
return false
}
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public init(content: Content) {
self.uuid = UUID()
self.content = content
self.referenceDrawingSize = .zero
self.position = CGPoint()
self.scale = 1.0
self.rotation = 0.0
self.mirrored = false
self.isExplicitlyStatic = false
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
if let gift = try container.decodeIfPresent(StarGift.UniqueGift.self, forKey: .gift) {
let size = try container.decodeIfPresent(CGSize.self, forKey: .messageSize) ?? .zero
self.content = .gift(gift, size)
} else if let messageIds = try container.decodeIfPresent([MessageId].self, forKey: .messageIds) {
let size = try container.decodeIfPresent(CGSize.self, forKey: .messageSize) ?? .zero
let file = try container.decodeIfPresent(TelegramMediaFile.self, forKey: .messageFile)
let mediaRect = try container.decodeIfPresent(CGRect.self, forKey: .messageMediaRect)
let mediaCornerRadius = try container.decodeIfPresent(CGFloat.self, forKey: .messageMediaCornerRadius)
self.content = .message(messageIds, size, file, mediaRect, mediaCornerRadius)
} else if let _ = try container.decodeIfPresent(Bool.self, forKey: .dualVideo) {
let isAdditional = try container.decodeIfPresent(Bool.self, forKey: .isAdditionalVideo) ?? false
self.content = .dualVideoReference(isAdditional)
} else if let file = try container.decodeIfPresent(TelegramMediaFile.self, forKey: .file) {
let fileType: Content.FileType
if let reaction = try container.decodeIfPresent(MessageReaction.Reaction.self, forKey: .reaction) {
var reactionStyle: Content.FileType.ReactionStyle = .white
if let style = try container.decodeIfPresent(Int32.self, forKey: .reactionStyle) {
reactionStyle = DrawingStickerEntity.Content.FileType.ReactionStyle(rawValue: style) ?? .white
}
fileType = .reaction(reaction, reactionStyle)
} else {
fileType = .sticker
}
self.content = .file(.standalone(media: file), fileType)
} else if let dataPath = try container.decodeIfPresent(String.self, forKey: .animatedImagePath), let data = try? Data(contentsOf: URL(fileURLWithPath: fullEntityMediaPath(dataPath))), let imagePath = try container.decodeIfPresent(String.self, forKey: .imagePath), let thumbnailImage = UIImage(contentsOfFile: fullEntityMediaPath(imagePath)) {
self.content = .animatedImage(data, thumbnailImage)
} else if let imagePath = try container.decodeIfPresent(String.self, forKey: .imagePath), let image = UIImage(contentsOfFile: fullEntityMediaPath(imagePath)) {
let isRectangle = try container.decodeIfPresent(Bool.self, forKey: .isRectangle) ?? false
let isDualPhoto = try container.decodeIfPresent(Bool.self, forKey: .isDualPhoto) ?? false
let imageType: Content.ImageType
if isDualPhoto {
imageType = .dualPhoto
} else if isRectangle {
imageType = .rectangle
} else {
imageType = .sticker
}
self.content = .image(image, imageType)
} else if let file = try container.decodeIfPresent(TelegramMediaFile.self, forKey: .videoFile) {
self.content = .video(file)
} else {
throw DrawingStickerEntity.DecodingError.generic
}
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.scale = try container.decode(CGFloat.self, forKey: .scale)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
self.mirrored = try container.decode(Bool.self, forKey: .mirrored)
self.isExplicitlyStatic = try container.decodeIfPresent(Bool.self, forKey: .isExplicitlyStatic) ?? false
self.canCutOut = try container.decodeIfPresent(Bool.self, forKey: .canCutOut) ?? false
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
if let renderSubEntities = try? container.decodeIfPresent([CodableDrawingEntity].self, forKey: .renderSubEntities) {
self.renderSubEntities = renderSubEntities.compactMap { $0.entity as? DrawingStickerEntity }
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
switch self.content {
case let .file(file, fileType):
try container.encode(file.media, forKey: .file)
switch fileType {
case let .reaction(reaction, reactionStyle):
try container.encode(reaction, forKey: .reaction)
try container.encode(reactionStyle.rawValue, forKey: .reactionStyle)
default:
break
}
case let .image(image, imageType):
let imagePath = "\(self.uuid).png"
let fullImagePath = fullEntityMediaPath(imagePath)
if let imageData = image.pngData() {
try? FileManager.default.createDirectory(atPath: entitiesPath(), withIntermediateDirectories: true)
try? imageData.write(to: URL(fileURLWithPath: fullImagePath))
try container.encodeIfPresent(imagePath, forKey: .imagePath)
}
switch imageType {
case .dualPhoto:
try container.encode(true, forKey: .isDualPhoto)
case .rectangle:
try container.encode(true, forKey: .isRectangle)
default:
break
}
case let .animatedImage(data, thumbnailImage):
let dataPath = "\(self.uuid).heics"
let fullDataPath = fullEntityMediaPath(dataPath)
try? FileManager.default.createDirectory(atPath: entitiesPath(), withIntermediateDirectories: true)
try? data.write(to: URL(fileURLWithPath: fullDataPath))
try container.encodeIfPresent(dataPath, forKey: .animatedImagePath)
let imagePath = "\(self.uuid).png"
let fullImagePath = fullEntityMediaPath(imagePath)
if let imageData = thumbnailImage.pngData() {
try? FileManager.default.createDirectory(atPath: entitiesPath(), withIntermediateDirectories: true)
try? imageData.write(to: URL(fileURLWithPath: fullImagePath))
try container.encodeIfPresent(imagePath, forKey: .imagePath)
}
case let .video(file):
try container.encode(file, forKey: .videoFile)
case let .dualVideoReference(isAdditional):
try container.encode(true, forKey: .dualVideo)
try container.encode(isAdditional, forKey: .isAdditionalVideo)
case let .message(messageIds, size, file, mediaRect, mediaCornerRadius):
try container.encode(messageIds, forKey: .messageIds)
try container.encode(size, forKey: .messageSize)
try container.encodeIfPresent(file, forKey: .messageFile)
try container.encodeIfPresent(mediaRect, forKey: .messageMediaRect)
try container.encodeIfPresent(mediaCornerRadius, forKey: .messageMediaCornerRadius)
case let .gift(gift, size):
try container.encode(gift, forKey: .gift)
try container.encode(size, forKey: .messageSize)
}
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.scale, forKey: .scale)
try container.encode(self.rotation, forKey: .rotation)
try container.encode(self.mirrored, forKey: .mirrored)
try container.encode(self.isExplicitlyStatic, forKey: .isExplicitlyStatic)
try container.encode(self.canCutOut, forKey: .canCutOut)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
if let renderSubEntities = self.renderSubEntities {
let codableEntities: [CodableDrawingEntity] = renderSubEntities.compactMap { CodableDrawingEntity(entity: $0) }
try container.encode(codableEntities, forKey: .renderSubEntities)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingStickerEntity(content: self.content)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.scale = self.scale
newEntity.rotation = self.rotation
newEntity.mirrored = self.mirrored
newEntity.isExplicitlyStatic = self.isExplicitlyStatic
newEntity.canCutOut = self.canCutOut
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingStickerEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.content != other.content {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.scale != other.scale {
return false
}
if self.rotation != other.rotation {
return false
}
if self.mirrored != other.mirrored {
return false
}
if self.isExplicitlyStatic != other.isExplicitlyStatic {
return false
}
if self.canCutOut != other.canCutOut {
return false
}
return true
}
}
public extension UIImage {
class func animatedImageFromData(data: Data) -> DrawingAnimatedImage? {
guard let source = CGImageSourceCreateWithData(data as CFData, nil) else {
return nil
}
let count = CGImageSourceGetCount(source)
var images = [UIImage]()
var duration = 0.0
for i in 0 ..< count {
if let cgImage = CGImageSourceCreateImageAtIndex(source, i, nil) {
let image = UIImage(cgImage: cgImage)
images.append(image)
let delaySeconds = UIImage.delayForImageAtIndex(Int(i), source: source)
duration += delaySeconds
}
}
return DrawingAnimatedImage(images: images, duration: duration)
}
class func delayForImageAtIndex(_ index: Int, source: CGImageSource!) -> Double {
var delay = 0.0
guard #available(iOS 13.0, *) else {
return delay
}
let cfProperties = CGImageSourceCopyPropertiesAtIndex(source, index, nil)
let gifPropertiesPointer = UnsafeMutablePointer<UnsafeRawPointer?>.allocate(capacity: 0)
if CFDictionaryGetValueIfPresent(cfProperties, Unmanaged.passUnretained(kCGImagePropertyHEICSDictionary).toOpaque(), gifPropertiesPointer) == false {
return delay
}
let gifProperties:CFDictionary = unsafeBitCast(gifPropertiesPointer.pointee, to: CFDictionary.self)
var delayObject: AnyObject = unsafeBitCast(CFDictionaryGetValue(gifProperties, Unmanaged.passUnretained(kCGImagePropertyHEICSUnclampedDelayTime).toOpaque()), to: AnyObject.self)
if delayObject.doubleValue == 0 {
delayObject = unsafeBitCast(CFDictionaryGetValue(gifProperties, Unmanaged.passUnretained(kCGImagePropertyHEICSDelayTime).toOpaque()), to: AnyObject.self)
}
delay = delayObject as? Double ?? 0
return delay
}
}
public final class DrawingAnimatedImage {
public let images: [UIImage]
public let duration: Double
init(images: [UIImage], duration: Double) {
self.images = images
self.duration = duration
}
}
@@ -0,0 +1,347 @@
import Foundation
import UIKit
import Display
import AccountContext
import TextFormat
public final class DrawingTextEntity: DrawingEntity, Codable {
final class CustomEmojiAttribute: Codable {
private enum CodingKeys: String, CodingKey {
case attribute
case rangeOrigin
case rangeLength
}
let attribute: ChatTextInputTextCustomEmojiAttribute
let range: NSRange
init(attribute: ChatTextInputTextCustomEmojiAttribute, range: NSRange) {
self.attribute = attribute
self.range = range
}
init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.attribute = try container.decode(ChatTextInputTextCustomEmojiAttribute.self, forKey: .attribute)
let rangeOrigin = try container.decode(Int.self, forKey: .rangeOrigin)
let rangeLength = try container.decode(Int.self, forKey: .rangeLength)
self.range = NSMakeRange(rangeOrigin, rangeLength)
}
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.attribute, forKey: .attribute)
try container.encode(self.range.location, forKey: .rangeOrigin)
try container.encode(self.range.length, forKey: .rangeLength)
}
}
private enum CodingKeys: String, CodingKey {
case uuid
case text
case textAttributes
case style
case animation
case font
case alignment
case fontSize
case color
case referenceDrawingSize
case position
case width
case scale
case rotation
case renderImage
case renderSubEntities
case renderAnimationFrames
}
public enum Style: Codable, Equatable {
case regular
case filled
case semi
case stroke
case blur
}
public enum Animation: Codable, Equatable {
case none
case typing
case wiggle
case zoomIn
}
public enum Font: Codable, Equatable {
case sanFrancisco
case other(String, String)
}
public enum Alignment: Codable, Equatable {
case left
case center
case right
}
public var uuid: UUID
public var isAnimated: Bool {
if self.animation != .none {
return true
}
var isAnimated = false
if let renderSubEntities = self.renderSubEntities {
for entity in renderSubEntities {
if entity.isAnimated {
isAnimated = true
break
}
}
}
return isAnimated
}
public struct TextAttributes {
public static let color = NSAttributedString.Key(rawValue: "Attribute__Color")
}
public var text: NSAttributedString
public var style: Style
public var animation: Animation
public var font: Font
public var alignment: Alignment
public var fontSize: CGFloat
public var color: DrawingColor
public var lineWidth: CGFloat = 0.0
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var width: CGFloat
public var scale: CGFloat
public var rotation: CGFloat
public var center: CGPoint {
return self.position
}
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public class AnimationFrame: Codable {
private enum CodingKeys: String, CodingKey {
case timestamp
case duration
case image
}
public let timestamp: Double
public let duration: Double
public let image: UIImage
public init(timestamp: Double, duration: Double, image: UIImage) {
self.timestamp = timestamp
self.duration = duration
self.image = image
}
required public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.timestamp = try container.decode(Double.self, forKey: .timestamp)
self.duration = try container.decode(Double.self, forKey: .duration)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .image) {
self.image = UIImage(data: renderImageData)!
} else {
fatalError()
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.timestamp, forKey: .timestamp)
try container.encode(self.duration, forKey: .duration)
if let data = self.image.pngData() {
try container.encode(data, forKey: .image)
}
}
}
public var renderAnimationFrames: [AnimationFrame]?
public init(text: NSAttributedString, style: Style, animation: Animation, font: Font, alignment: Alignment, fontSize: CGFloat, color: DrawingColor) {
self.uuid = UUID()
self.text = text
self.style = style
self.animation = animation
self.font = font
self.alignment = alignment
self.fontSize = fontSize
self.color = color
self.referenceDrawingSize = .zero
self.position = .zero
self.width = 100.0
self.scale = 1.0
self.rotation = 0.0
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
let text = try container.decode(String.self, forKey: .text)
let attributedString = NSMutableAttributedString(string: text)
let textAttributes = try container.decode([CustomEmojiAttribute].self, forKey: .textAttributes)
for attribute in textAttributes {
attributedString.addAttribute(ChatTextInputAttributes.customEmoji, value: attribute.attribute, range: attribute.range)
}
self.text = attributedString
self.style = try container.decode(Style.self, forKey: .style)
self.animation = try container.decode(Animation.self, forKey: .animation)
self.font = try container.decode(Font.self, forKey: .font)
self.alignment = try container.decode(Alignment.self, forKey: .alignment)
self.fontSize = try container.decode(CGFloat.self, forKey: .fontSize)
self.color = try container.decode(DrawingColor.self, forKey: .color)
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.width = try container.decode(CGFloat.self, forKey: .width)
self.scale = try container.decode(CGFloat.self, forKey: .scale)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
if let renderSubEntities = try? container.decodeIfPresent([CodableDrawingEntity].self, forKey: .renderSubEntities) {
self.renderSubEntities = renderSubEntities.compactMap { $0.entity as? DrawingStickerEntity }
}
self.renderAnimationFrames = try container.decodeIfPresent([AnimationFrame].self, forKey: .renderAnimationFrames)
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.text.string, forKey: .text)
var textAttributes: [CustomEmojiAttribute] = []
self.text.enumerateAttributes(in: NSMakeRange(0, self.text.length), options: [], using: { attributes, range, _ in
if let value = attributes[ChatTextInputAttributes.customEmoji] as? ChatTextInputTextCustomEmojiAttribute {
textAttributes.append(CustomEmojiAttribute(attribute: value, range: range))
}
})
try container.encode(textAttributes, forKey: .textAttributes)
try container.encode(self.style, forKey: .style)
try container.encode(self.animation, forKey: .animation)
try container.encode(self.font, forKey: .font)
try container.encode(self.alignment, forKey: .alignment)
try container.encode(self.fontSize, forKey: .fontSize)
try container.encode(self.color, forKey: .color)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.width, forKey: .width)
try container.encode(self.scale, forKey: .scale)
try container.encode(self.rotation, forKey: .rotation)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
if let renderSubEntities = self.renderSubEntities {
let codableEntities: [CodableDrawingEntity] = renderSubEntities.compactMap { CodableDrawingEntity(entity: $0) }
try container.encode(codableEntities, forKey: .renderSubEntities)
}
if let renderAnimationFrames = self.renderAnimationFrames {
try container.encode(renderAnimationFrames, forKey: .renderAnimationFrames)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingTextEntity(text: self.text, style: self.style, animation: self.animation, font: self.font, alignment: self.alignment, fontSize: self.fontSize, color: self.color)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.width = self.width
newEntity.scale = self.scale
newEntity.rotation = self.rotation
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingTextEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.text != other.text {
return false
}
if self.style != other.style {
return false
}
if self.animation != other.animation {
return false
}
if self.font != other.font {
return false
}
if self.alignment != other.alignment {
return false
}
if self.fontSize != other.fontSize {
return false
}
if self.color != other.color {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.width != other.width {
return false
}
if self.scale != other.scale {
return false
}
if self.rotation != other.rotation {
return false
}
return true
}
}
public extension DrawingTextEntity {
func setColor(_ color: DrawingColor, range: NSRange) {
if range.length == 0 {
self.color = color
let updatedText = self.text.mutableCopy() as! NSMutableAttributedString
let range = NSMakeRange(0, updatedText.length)
updatedText.removeAttribute(DrawingTextEntity.TextAttributes.color, range: range)
self.text = updatedText
} else {
let updatedText = self.text.mutableCopy() as! NSMutableAttributedString
updatedText.removeAttribute(DrawingTextEntity.TextAttributes.color, range: range)
updatedText.addAttribute(DrawingTextEntity.TextAttributes.color, value: color.toUIColor(), range: range)
self.text = updatedText
}
}
func color(in range: NSRange) -> DrawingColor {
if range.length == 0 {
return self.color
} else {
if let color = self.text.attribute(DrawingTextEntity.TextAttributes.color, at: range.location, effectiveRange: nil) as? UIColor {
return DrawingColor(color: color)
} else {
return self.color
}
}
}
}
@@ -0,0 +1,147 @@
import Foundation
import UIKit
import Display
import AccountContext
public final class DrawingVectorEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case type
case color
case lineWidth
case drawingSize
case referenceDrawingSize
case start
case mid
case end
case renderImage
}
public enum VectorType: Codable {
case line
case oneSidedArrow
case twoSidedArrow
}
public var uuid: UUID
public let isAnimated: Bool
public var type: VectorType
public var color: DrawingColor
public var lineWidth: CGFloat
public var drawingSize: CGSize
public var referenceDrawingSize: CGSize
public var start: CGPoint
public var mid: (CGFloat, CGFloat)
public var end: CGPoint
public var center: CGPoint {
return self.start
}
public var scale: CGFloat = 1.0
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public init(type: VectorType, color: DrawingColor, lineWidth: CGFloat) {
self.uuid = UUID()
self.isAnimated = false
self.type = type
self.color = color
self.lineWidth = lineWidth
self.drawingSize = .zero
self.referenceDrawingSize = .zero
self.start = CGPoint()
self.mid = (0.5, 0.0)
self.end = CGPoint()
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.isAnimated = false
self.type = try container.decode(VectorType.self, forKey: .type)
self.color = try container.decode(DrawingColor.self, forKey: .color)
self.lineWidth = try container.decode(CGFloat.self, forKey: .lineWidth)
self.drawingSize = try container.decode(CGSize.self, forKey: .drawingSize)
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.start = try container.decode(CGPoint.self, forKey: .start)
let mid = try container.decode(CGPoint.self, forKey: .mid)
self.mid = (mid.x, mid.y)
self.end = try container.decode(CGPoint.self, forKey: .end)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.type, forKey: .type)
try container.encode(self.color, forKey: .color)
try container.encode(self.lineWidth, forKey: .lineWidth)
try container.encode(self.drawingSize, forKey: .drawingSize)
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.start, forKey: .start)
try container.encode(CGPoint(x: self.mid.0, y: self.mid.1), forKey: .mid)
try container.encode(self.end, forKey: .end)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingVectorEntity(type: self.type, color: self.color, lineWidth: self.lineWidth)
if copy {
newEntity.uuid = self.uuid
}
newEntity.drawingSize = self.drawingSize
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.start = self.start
newEntity.mid = self.mid
newEntity.end = self.end
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingVectorEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.type != other.type {
return false
}
if self.color != other.color {
return false
}
if self.lineWidth != other.lineWidth {
return false
}
if self.drawingSize != other.drawingSize {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.start != other.start {
return false
}
if self.mid.0 != other.mid.0 || self.mid.1 != other.mid.1 {
return false
}
if self.end != other.end {
return false
}
return true
}
}
@@ -0,0 +1,193 @@
import Foundation
import UIKit
import Display
import AccountContext
import TextFormat
import Postbox
import TelegramCore
public final class DrawingWeatherEntity: DrawingEntity, Codable {
private enum CodingKeys: String, CodingKey {
case uuid
case style
case color
case hasCustomColor
case emoji
case temperature
case icon
case referenceDrawingSize
case position
case width
case scale
case rotation
case renderImage
}
public enum Style: Codable, Equatable {
case white
case black
case transparent
case custom
}
public var uuid: UUID
public var isAnimated: Bool {
return false
}
public var style: Style
public var icon: TelegramMediaFile?
public var emoji: String
public var temperature: Double
public var color: DrawingColor = DrawingColor(color: .white) {
didSet {
if self.color.toUIColor().argb == UIColor.white.argb {
self.style = .white
self.hasCustomColor = false
} else {
self.style = .custom
self.hasCustomColor = true
}
}
}
public var hasCustomColor = false
public var lineWidth: CGFloat = 0.0
public var referenceDrawingSize: CGSize
public var position: CGPoint
public var width: CGFloat
public var scale: CGFloat {
didSet {
self.scale = min(2.5, self.scale)
}
}
public var rotation: CGFloat
public var center: CGPoint {
return self.position
}
public var renderImage: UIImage?
public var renderSubEntities: [DrawingEntity]?
public var isMedia: Bool {
return false
}
public init(emoji: String, emojiFile: TelegramMediaFile?, temperature: Double, style: Style) {
self.uuid = UUID()
self.emoji = emoji
self.icon = emojiFile
self.temperature = temperature
self.style = style
self.referenceDrawingSize = .zero
self.position = .zero
self.width = 100.0
self.scale = 1.0
self.rotation = 0.0
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.uuid = try container.decode(UUID.self, forKey: .uuid)
self.emoji = try container.decode(String.self, forKey: .emoji)
self.temperature = try container.decode(Double.self, forKey: .temperature)
self.style = try container.decode(Style.self, forKey: .style)
self.color = try container.decodeIfPresent(DrawingColor.self, forKey: .color) ?? DrawingColor(color: .white)
self.hasCustomColor = try container.decodeIfPresent(Bool.self, forKey: .hasCustomColor) ?? false
if let iconData = try container.decodeIfPresent(Data.self, forKey: .icon) {
self.icon = PostboxDecoder(buffer: MemoryBuffer(data: iconData)).decodeRootObject() as? TelegramMediaFile
}
self.referenceDrawingSize = try container.decode(CGSize.self, forKey: .referenceDrawingSize)
self.position = try container.decode(CGPoint.self, forKey: .position)
self.width = try container.decode(CGFloat.self, forKey: .width)
self.scale = try container.decode(CGFloat.self, forKey: .scale)
self.rotation = try container.decode(CGFloat.self, forKey: .rotation)
if let renderImageData = try? container.decodeIfPresent(Data.self, forKey: .renderImage) {
self.renderImage = UIImage(data: renderImageData)
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.uuid, forKey: .uuid)
try container.encode(self.emoji, forKey: .emoji)
try container.encode(self.temperature, forKey: .temperature)
try container.encode(self.style, forKey: .style)
try container.encode(self.color, forKey: .color)
try container.encode(self.hasCustomColor, forKey: .hasCustomColor)
var encoder = PostboxEncoder()
if let icon = self.icon {
encoder = PostboxEncoder()
encoder.encodeRootObject(icon)
let iconData = encoder.makeData()
try container.encode(iconData, forKey: .icon)
}
try container.encode(self.referenceDrawingSize, forKey: .referenceDrawingSize)
try container.encode(self.position, forKey: .position)
try container.encode(self.width, forKey: .width)
try container.encode(self.scale, forKey: .scale)
try container.encode(self.rotation, forKey: .rotation)
if let renderImage, let data = renderImage.pngData() {
try container.encode(data, forKey: .renderImage)
}
}
public func duplicate(copy: Bool) -> DrawingEntity {
let newEntity = DrawingWeatherEntity(emoji: self.emoji, emojiFile: self.icon, temperature: self.temperature, style: self.style)
if copy {
newEntity.uuid = self.uuid
}
newEntity.referenceDrawingSize = self.referenceDrawingSize
newEntity.position = self.position
newEntity.width = self.width
newEntity.scale = self.scale
newEntity.rotation = self.rotation
return newEntity
}
public func isEqual(to other: DrawingEntity) -> Bool {
guard let other = other as? DrawingWeatherEntity else {
return false
}
if self.uuid != other.uuid {
return false
}
if self.emoji != other.emoji {
return false
}
if self.temperature != other.temperature {
return false
}
if self.style != other.style {
return false
}
if self.color != other.color {
return false
}
if self.referenceDrawingSize != other.referenceDrawingSize {
return false
}
if self.position != other.position {
return false
}
if self.width != other.width {
return false
}
if self.scale != other.scale {
return false
}
if self.rotation != other.rotation {
return false
}
return true
}
}
@@ -0,0 +1,443 @@
import Foundation
import UIKit
import AsyncDisplayKit
import Display
import SwiftSignalKit
import Postbox
import TelegramCore
import TelegramPresentationData
import AccountContext
import WallpaperBackgroundNode
public final class DrawingWallpaperRenderer {
private let context: AccountContext
private let dayWallpaper: TelegramWallpaper?
private let nightWallpaper: TelegramWallpaper?
private let wallpaperBackgroundNode: WallpaperBackgroundNode
private let darkWallpaperBackgroundNode: WallpaperBackgroundNode
public init (context: AccountContext, dayWallpaper: TelegramWallpaper?, nightWallpaper: TelegramWallpaper?) {
self.context = context
self.dayWallpaper = dayWallpaper
self.nightWallpaper = nightWallpaper
self.wallpaperBackgroundNode = createWallpaperBackgroundNode(context: context, forChatDisplay: true, useSharedAnimationPhase: false)
self.wallpaperBackgroundNode.displaysAsynchronously = false
let wallpaper = self.dayWallpaper ?? context.sharedContext.currentPresentationData.with { $0 }.chatWallpaper
self.wallpaperBackgroundNode.update(wallpaper: wallpaper, animated: false)
self.darkWallpaperBackgroundNode = createWallpaperBackgroundNode(context: context, forChatDisplay: true, useSharedAnimationPhase: false)
self.darkWallpaperBackgroundNode.displaysAsynchronously = false
let darkTheme = defaultDarkColorPresentationTheme
let darkWallpaper = self.nightWallpaper ?? darkTheme.chat.defaultWallpaper
self.darkWallpaperBackgroundNode.update(wallpaper: darkWallpaper, animated: false)
}
public func render(completion: @escaping (CGSize, UIImage?, UIImage?, CGRect?) -> Void) {
self.updateLayout(size: CGSize(width: 360.0, height: 640.0))
let resultSize = CGSize(width: 1080, height: 1920)
Queue.mainQueue().justDispatch {
self.generate(view: self.wallpaperBackgroundNode.view) { dayImage in
if self.dayWallpaper != nil && self.nightWallpaper == nil {
completion(resultSize, dayImage, nil, nil)
} else {
Queue.mainQueue().justDispatch {
self.generate(view: self.darkWallpaperBackgroundNode.view) { nightImage in
completion(resultSize, dayImage, nightImage, nil)
}
}
}
}
}
}
private func generate(view: UIView, completion: @escaping (UIImage) -> Void) {
let size = CGSize(width: 360.0, height: 640.0)
UIGraphicsBeginImageContextWithOptions(size, false, 3.0)
view.drawHierarchy(in: CGRect(origin: CGPoint(), size: size), afterScreenUpdates: true)
let img = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
let finalImage = generateImage(CGSize(width: size.width * 3.0, height: size.height * 3.0), contextGenerator: { size, context in
if let cgImage = img?.cgImage {
context.draw(cgImage, in: CGRect(origin: .zero, size: size), byTiling: false)
}
}, opaque: true, scale: 1.0)
if let finalImage {
completion(finalImage)
}
}
private func updateLayout(size: CGSize) {
self.wallpaperBackgroundNode.updateLayout(size: size, displayMode: .aspectFill, transition: .immediate)
self.wallpaperBackgroundNode.frame = CGRect(origin: .zero, size: size)
self.darkWallpaperBackgroundNode.updateLayout(size: size, displayMode: .aspectFill, transition: .immediate)
self.darkWallpaperBackgroundNode.frame = CGRect(origin: .zero, size: size)
}
}
public final class DrawingMessageRenderer {
final class ContainerNode: ASDisplayNode {
private let context: AccountContext
private let messages: [Message]
private let isNight: Bool
private let isOverlay: Bool
private let isLink: Bool
private let isGift: Bool
private let wallpaperColor: UIColor?
private let messagesContainerNode: ASDisplayNode
private var avatarHeaderNode: ListViewItemHeaderNode?
private var messageNodes: [ListViewItemNode]?
init(
context: AccountContext,
messages: [Message],
isNight: Bool = false,
isOverlay: Bool = false,
isLink: Bool = false,
isGift: Bool = false,
wallpaperColor: UIColor? = nil
) {
self.context = context
self.messages = messages
self.isNight = isNight
self.isOverlay = isOverlay
self.isLink = isLink
self.isGift = isGift
self.wallpaperColor = wallpaperColor
self.messagesContainerNode = ASDisplayNode()
self.messagesContainerNode.displaysAsynchronously = false
self.messagesContainerNode.clipsToBounds = true
self.messagesContainerNode.transform = CATransform3DMakeScale(1.0, -1.0, 1.0)
super.init()
self.displaysAsynchronously = false
self.addSubnode(self.messagesContainerNode)
}
public func render(presentationData: PresentationData, completion: @escaping (CGSize, UIImage?, CGRect?) -> Void) {
var mockPresentationData = presentationData
if self.isNight {
let darkTheme = defaultDarkColorPresentationTheme
mockPresentationData = mockPresentationData.withUpdated(theme: darkTheme).withUpdated(chatWallpaper: darkTheme.chat.defaultWallpaper)
}
let layout = ContainerViewLayout(size: CGSize(width: 360.0, height: 640.0), metrics: LayoutMetrics(widthClass: .compact, heightClass: .compact, orientation: .portrait), deviceMetrics: .iPhoneX, intrinsicInsets: .zero, safeInsets: .zero, additionalInsets: .zero, statusBarHeight: 0.0, inputHeight: nil, inputHeightIsInteractivellyChanging: false, inVoiceOver: false)
let size = self.updateMessagesLayout(layout: layout, presentationData: mockPresentationData)
let _ = self.updateMessagesLayout(layout: layout, presentationData: mockPresentationData)
Queue.mainQueue().after(0.35, {
var mediaRect: CGRect?
if let messageNode = self.messageNodes?.first {
if self.isOverlay {
func hideNonOverlayViews(_ view: UIView) -> Bool {
var hasResult = false
for view in view.subviews {
if view.tag == 0xFACE {
hasResult = true
} else {
if hideNonOverlayViews(view) {
hasResult = true
} else {
view.isHidden = true
}
}
}
return hasResult
}
let _ = hideNonOverlayViews(messageNode.view)
} else if !self.isNight {
func findMediaView(_ view: UIView) -> UIView? {
for view in view.subviews {
if let _ = view.asyncdisplaykit_node as? UniversalVideoNode {
return view
} else {
if let result = findMediaView(view) {
return result
}
}
}
return nil
}
if let mediaView = findMediaView(messageNode.view) {
var rect = mediaView.convert(mediaView.bounds, to: self.messagesContainerNode.view)
rect.origin.y = self.messagesContainerNode.frame.height - rect.maxY
mediaRect = rect
}
}
}
var borderColor: UIColor?
if self.isGift && !self.isOverlay, let wallpaperColor = self.wallpaperColor {
borderColor = wallpaperColor.withMultiplied(hue: 1.0, saturation: 1.5, brightness: self.isNight ? 1.6 : 0.7).withAlphaComponent(0.6)
}
self.generate(size: size, borderColor: borderColor) { image in
completion(size, image, mediaRect)
}
})
}
private func generate(size: CGSize, borderColor: UIColor? = nil, completion: @escaping (UIImage) -> Void) {
UIGraphicsBeginImageContextWithOptions(size, false, 3.0)
self.view.drawHierarchy(in: CGRect(origin: CGPoint(), size: size), afterScreenUpdates: true)
let img = UIGraphicsGetImageFromCurrentImageContext()
UIGraphicsEndImageContext()
let finalImage = generateImage(CGSize(width: size.width * 3.0, height: size.height * 3.0), contextGenerator: { size, context in
context.clear(CGRect(origin: .zero, size: size))
if let borderColor {
context.addPath(CGPath(roundedRect: CGRect(origin: CGPoint(x: 6.0, y: 12.0), size: CGSize(width: size.width - 6.0, height: size.height - 13.0)), cornerWidth: 70.0, cornerHeight: 70.0, transform: nil))
context.setFillColor(borderColor.cgColor)
context.fillPath()
}
if let cgImage = img?.cgImage {
context.draw(cgImage, in: CGRect(origin: .zero, size: size), byTiling: false)
}
}, opaque: false, scale: 1.0)
if let finalImage {
completion(finalImage)
}
}
private func updateMessagesLayout(layout: ContainerViewLayout, presentationData: PresentationData) -> CGSize {
let size = layout.size
let theme = presentationData.theme.withUpdated(preview: true)
let chatBubbleCorners = PresentationChatBubbleCorners(
mainRadius: presentationData.chatBubbleCorners.mainRadius,
auxiliaryRadius: presentationData.chatBubbleCorners.auxiliaryRadius,
mergeBubbleCorners: presentationData.chatBubbleCorners.mergeBubbleCorners,
hasTails: !self.isLink
)
let avatarHeaderItem: ListViewItemHeader?
if let author = self.messages.first?.author {
let avatarPeer: Peer
if let peer = self.messages.first!.peers[author.id] {
avatarPeer = peer
} else {
avatarPeer = author
}
avatarHeaderItem = self.context.sharedContext.makeChatMessageAvatarHeaderItem(context: self.context, timestamp: self.messages.first?.timestamp ?? 0, peer: avatarPeer, message: self.messages.first!, theme: theme, strings: presentationData.strings, wallpaper: presentationData.chatWallpaper, fontSize: presentationData.chatFontSize, chatBubbleCorners: chatBubbleCorners, dateTimeFormat: presentationData.dateTimeFormat, nameOrder: presentationData.nameDisplayOrder)
} else {
avatarHeaderItem = nil
}
let items: [ListViewItem] = [self.context.sharedContext.makeChatMessagePreviewItem(context: self.context, messages: self.messages, theme: theme, strings: presentationData.strings, wallpaper: presentationData.theme.chat.defaultWallpaper, fontSize: presentationData.chatFontSize, chatBubbleCorners: chatBubbleCorners, dateTimeFormat: presentationData.dateTimeFormat, nameOrder: presentationData.nameDisplayOrder, forcedResourceStatus: nil, tapMessage: nil, clickThroughMessage: nil, backgroundNode: nil, availableReactions: nil, accountPeer: nil, isCentered: false, isPreview: true, isStandalone: false)]
let inset: CGFloat = 16.0
var leftInset: CGFloat = 37.0
if self.isLink {
leftInset = -6.0
} else if self.isGift {
leftInset = -50.0
}
let containerWidth = layout.size.width - inset * 2.0
let params = ListViewItemLayoutParams(width: containerWidth, leftInset: layout.safeInsets.left, rightInset: layout.safeInsets.right, availableHeight: layout.size.height)
var width: CGFloat = containerWidth
var height: CGFloat = size.height
if let messageNodes = self.messageNodes {
for i in 0 ..< items.count {
let itemNode = messageNodes[i]
items[i].updateNode(async: { $0() }, node: {
return itemNode
}, params: params, previousItem: i == 0 ? nil : items[i - 1], nextItem: i == (items.count - 1) ? nil : items[i + 1], animation: .None, completion: { (layout, apply) in
let nodeFrame = CGRect(origin: CGPoint(x: 0.0, y: 0.0), size: CGSize(width: containerWidth, height: layout.size.height))
itemNode.contentSize = layout.contentSize
itemNode.insets = layout.insets
itemNode.frame = nodeFrame
itemNode.isUserInteractionEnabled = false
apply(ListViewItemApply(isOnScreen: true))
})
}
} else {
var messageNodes: [ListViewItemNode] = []
for i in 0 ..< items.count {
var itemNode: ListViewItemNode?
items[i].nodeConfiguredForParams(async: { $0() }, params: params, synchronousLoads: true, previousItem: i == 0 ? nil : items[i - 1], nextItem: i == (items.count - 1) ? nil : items[i + 1], completion: { node, apply in
itemNode = node
apply().1(ListViewItemApply(isOnScreen: true))
})
itemNode!.subnodeTransform = CATransform3DMakeScale(-1.0, 1.0, 1.0)
itemNode!.isUserInteractionEnabled = false
messageNodes.append(itemNode!)
self.messagesContainerNode.addSubnode(itemNode!)
}
self.messageNodes = messageNodes
}
if let messageNodes = self.messageNodes {
var minX: CGFloat = .greatestFiniteMagnitude
var maxX: CGFloat = -.greatestFiniteMagnitude
var minY: CGFloat = .greatestFiniteMagnitude
var maxY: CGFloat = -.greatestFiniteMagnitude
for node in messageNodes {
if node.frame.minY < minY {
minY = node.frame.minY
}
if node.frame.maxY > maxY {
maxY = node.frame.maxY
}
if let areaNode = node.subnodes?.last {
if areaNode.frame.minX < minX {
minX = areaNode.frame.minX
}
if areaNode.frame.maxX > maxX {
maxX = areaNode.frame.maxX
}
}
}
width = abs(maxX - minX)
height = abs(maxY - minY)
}
var bottomOffset: CGFloat = 0.0
if let messageNodes = self.messageNodes {
for itemNode in messageNodes {
itemNode.frame = CGRect(origin: CGPoint(x: leftInset, y: 0.0), size: itemNode.frame.size)
bottomOffset += itemNode.frame.maxY
itemNode.updateFrame(itemNode.frame, within: layout.size)
}
}
if let avatarHeaderItem {
let avatarHeaderNode: ListViewItemHeaderNode
if let currentAvatarHeaderNode = self.avatarHeaderNode {
avatarHeaderNode = currentAvatarHeaderNode
avatarHeaderItem.updateNode(avatarHeaderNode, previous: nil, next: avatarHeaderItem)
} else {
avatarHeaderNode = avatarHeaderItem.node(synchronousLoad: true)
avatarHeaderNode.subnodeTransform = CATransform3DMakeScale(-1.0, 1.0, 1.0)
self.messagesContainerNode.addSubnode(avatarHeaderNode)
self.avatarHeaderNode = avatarHeaderNode
}
avatarHeaderNode.frame = CGRect(origin: CGPoint(x: 0.0, y: 3.0), size: CGSize(width: layout.size.width, height: avatarHeaderItem.height))
avatarHeaderNode.updateLayout(size: size, leftInset: layout.safeInsets.left, rightInset: layout.safeInsets.right, transition: .immediate)
}
var finalWidth: CGFloat = width
if leftInset > 0.0 {
finalWidth += leftInset + 6.0
}
let containerSize = CGSize(width: finalWidth, height: height)
self.frame = CGRect(origin: CGPoint(x: -1000.0, y: 0.0), size: containerSize)
self.messagesContainerNode.frame = CGRect(origin: CGPoint(), size: containerSize)
return containerSize
}
}
public struct Result {
public struct MediaFrame {
public let rect: CGRect
public let cornerRadius: CGFloat
}
public let size: CGSize
public let dayImage: UIImage
public let nightImage: UIImage
public let overlayImage: UIImage
public let mediaFrame: MediaFrame?
}
private let context: AccountContext
private let messages: [Message]
private let dayContainerNode: ContainerNode
private let nightContainerNode: ContainerNode
private let overlayContainerNode: ContainerNode
public init(
context: AccountContext,
messages: [Message],
parentView: UIView,
isLink: Bool = false,
isGift: Bool = false,
wallpaperDayColor: UIColor? = nil,
wallpaperNightColor: UIColor? = nil
) {
self.context = context
self.messages = messages
self.dayContainerNode = ContainerNode(context: context, messages: messages, isLink: isLink, isGift: isGift, wallpaperColor: wallpaperDayColor)
self.nightContainerNode = ContainerNode(context: context, messages: messages, isNight: true, isLink: isLink, isGift: isGift, wallpaperColor: wallpaperNightColor)
self.overlayContainerNode = ContainerNode(context: context, messages: messages, isOverlay: true, isLink: isLink, isGift: isGift, wallpaperColor: nil)
parentView.addSubview(self.dayContainerNode.view)
parentView.addSubview(self.nightContainerNode.view)
parentView.addSubview(self.overlayContainerNode.view)
}
public func render(completion: @escaping (Result) -> Void) {
Queue.mainQueue().justDispatch {
let presentationData = self.context.sharedContext.currentPresentationData.with { $0 }
let defaultPresentationData = defaultPresentationData()
let mockPresentationData = PresentationData(
strings: presentationData.strings,
theme: defaultPresentationTheme,
autoNightModeTriggered: false,
chatWallpaper: presentationData.chatWallpaper,
chatFontSize: defaultPresentationData.chatFontSize,
chatBubbleCorners: defaultPresentationData.chatBubbleCorners,
listsFontSize: defaultPresentationData.listsFontSize,
dateTimeFormat: presentationData.dateTimeFormat,
nameDisplayOrder: presentationData.nameDisplayOrder,
nameSortOrder: presentationData.nameSortOrder,
reduceMotion: false,
largeEmoji: true
)
var finalSize: CGSize = .zero
var dayImage: UIImage?
var nightImage: UIImage?
var overlayImage: UIImage?
var mediaRect: CGRect?
let completeIfReady = {
if let dayImage, let nightImage, let overlayImage {
var cornerRadius: CGFloat = defaultPresentationData.chatBubbleCorners.mainRadius
if let mediaRect, mediaRect.width == mediaRect.height, mediaRect.width == 240.0 {
cornerRadius = mediaRect.width / 2.0
} else if let rect = mediaRect {
mediaRect = CGRect(x: rect.minX + 4.0, y: rect.minY, width: rect.width - 6.0, height: rect.height - 1.0)
}
completion(Result(size: finalSize, dayImage: dayImage, nightImage: nightImage, overlayImage: overlayImage, mediaFrame: mediaRect.flatMap { Result.MediaFrame(rect: $0, cornerRadius: cornerRadius) }))
self.dayContainerNode.view.removeFromSuperview()
self.nightContainerNode.view.removeFromSuperview()
self.overlayContainerNode.view.removeFromSuperview()
}
}
self.dayContainerNode.render(presentationData: mockPresentationData) { size, image, rect in
finalSize = size
dayImage = image
mediaRect = rect
completeIfReady()
}
self.nightContainerNode.render(presentationData: mockPresentationData) { size, image, _ in
nightImage = image
completeIfReady()
}
self.overlayContainerNode.render(presentationData: mockPresentationData) { size, image, _ in
overlayImage = image
completeIfReady()
}
}
}
}
@@ -0,0 +1,284 @@
import Foundation
import UIKit
import Metal
import MetalPerformanceShaders
import simd
import CoreImage
struct TextureSize {
let width: Int
let height: Int
}
private final class EnhanceLightnessPass: DefaultRenderPass {
fileprivate var cachedTexture: MTLTexture?
override var fragmentShaderFunctionName: String {
return "rgbToLightnessFragmentShader"
}
override var pixelFormat: MTLPixelFormat {
return .r8Unorm
}
func process(input: MTLTexture, size: TextureSize, scale: simd_float2, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device)
let width = size.width
let height = size.height
if self.cachedTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = width
textureDescriptor.height = height
textureDescriptor.pixelFormat = .r8Unorm
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return nil
}
texture.label = "lightnessTexture"
self.cachedTexture = texture
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return nil
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(size.width), height: Double(size.height),
znear: -1.0, zfar: 1.0)
)
let samplerDescriptor = MTLSamplerDescriptor()
samplerDescriptor.minFilter = .linear
samplerDescriptor.magFilter = .linear
samplerDescriptor.sAddressMode = .mirrorRepeat
samplerDescriptor.tAddressMode = .mirrorRepeat
samplerDescriptor.rAddressMode = .mirrorRepeat
guard let samplerState = device.makeSamplerState(descriptor: samplerDescriptor) else {
return nil
}
var scale = scale
renderCommandEncoder.setFragmentTexture(input, index: 0)
renderCommandEncoder.setFragmentBytes(&scale, length: MemoryLayout<simd_float2>.size, index: 0)
renderCommandEncoder.setFragmentSamplerState(samplerState, index: 0)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return self.cachedTexture!
}
}
private let binCount = 256
struct MediaEditorEnhanceLUTGeneratorParameters {
var histogramBins: simd_uint1
var clipLimit: simd_uint1
var totalPixelCountPerTile: simd_uint1
var numberOfLUTs: simd_uint1
}
private final class EnhanceLUTGeneratorPass: RenderPass {
fileprivate var pipelineState: MTLComputePipelineState?
fileprivate var histogramBuffer: MTLBuffer?
fileprivate var calculation: MPSImageHistogram?
private var lutTexture: MTLTexture?
func setup(device: MTLDevice, library: MTLLibrary) {
}
func setup(gridSize: TextureSize, device: MTLDevice, library: MTLLibrary) {
var histogramInfo = MPSImageHistogramInfo(
numberOfHistogramEntries: binCount,
histogramForAlpha: false,
minPixelValue: vector_float4(0,0,0,0),
maxPixelValue: vector_float4(1,1,1,1)
)
let calculation = MPSImageHistogram(device: device, histogramInfo: &histogramInfo)
calculation.zeroHistogram = false
self.calculation = calculation
let pipelineDescriptor = MTLComputePipelineDescriptor()
pipelineDescriptor.computeFunction = library.makeFunction(name: "enhanceGenerateLUT")
do {
self.pipelineState = try device.makeComputePipelineState(descriptor: pipelineDescriptor, options: .argumentInfo, reflection: nil)
} catch {
print(error.localizedDescription)
}
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
return nil
}
func process(input: MTLTexture, gridSize: TextureSize, clipLimit: Float, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
let lutCount = gridSize.width * gridSize.height
let tileSize = TextureSize(width: input.width / gridSize.width, height: input.height / gridSize.height);
let clipLimitValue = max(1, clipLimit * Float(tileSize.width * tileSize.height) / Float(binCount))
if self.lutTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = binCount
textureDescriptor.height = lutCount
textureDescriptor.pixelFormat = .r8Unorm
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return nil
}
self.lutTexture = texture
texture.label = "lutTexture"
}
guard let calculation = self.calculation, let histogramBuffer = device.makeBuffer(length: calculation.histogramSize(forSourceFormat: .r8Unorm) * lutCount, options: [.storageModePrivate]) else {
return nil
}
let histogramSize = calculation.histogramSize(forSourceFormat: input.pixelFormat)
for i in 0 ..< lutCount {
let col = i % gridSize.width
let row = i / gridSize.width
calculation.clipRectSource = MTLRegionMake2D(col * tileSize.width, row * tileSize.height, tileSize.width, tileSize.height)
calculation.encode(to: commandBuffer, sourceTexture: input, histogram: histogramBuffer, histogramOffset: i * histogramSize)
}
guard let computeCommandEncoder = commandBuffer.makeComputeCommandEncoder() else {
return nil
}
guard let pipelineState = self.pipelineState else {
return nil
}
var parameters = MediaEditorEnhanceLUTGeneratorParameters(
histogramBins: UInt32(binCount),
clipLimit: UInt32(clipLimitValue),
totalPixelCountPerTile: UInt32(tileSize.width * tileSize.height),
numberOfLUTs: UInt32(lutCount)
)
computeCommandEncoder.setComputePipelineState(pipelineState)
computeCommandEncoder.setBuffer(histogramBuffer, offset: 0, index: 0)
computeCommandEncoder.setBytes(&parameters, length: MemoryLayout<MediaEditorEnhanceLUTGeneratorParameters>.size, index: 1)
computeCommandEncoder.setTexture(self.lutTexture, index: 0)
let w = pipelineState.threadExecutionWidth
let threadsPerThreadgroup = MTLSize(width: w, height: 1, depth: 1)
let threadgroupsPerGrid = MTLSize(width: (lutCount + w - 1) / w, height: 1, depth: 1)
computeCommandEncoder.dispatchThreadgroups(threadgroupsPerGrid, threadsPerThreadgroup: threadsPerThreadgroup)
computeCommandEncoder.endEncoding()
return self.lutTexture!
}
}
private final class EnhanceLookupPass: DefaultRenderPass {
fileprivate var cachedTexture: MTLTexture?
override var fragmentShaderFunctionName: String {
return "enhanceColorLookupFragmentShader"
}
func process(input: MTLTexture, lookupTexture: MTLTexture, value: simd_float1, gridSize: simd_float2, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device)
let width = input.width
let height = input.height
if self.cachedTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = width
textureDescriptor.height = height
textureDescriptor.pixelFormat = input.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return input
}
self.cachedTexture = texture
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(width), height: Double(height),
znear: -1.0, zfar: 1.0)
)
var gridSize = gridSize
var value = value
renderCommandEncoder.setFragmentTexture(input, index: 0)
renderCommandEncoder.setFragmentTexture(lookupTexture, index: 1)
renderCommandEncoder.setFragmentBytes(&gridSize, length: MemoryLayout<simd_float2>.size, index: 0)
renderCommandEncoder.setFragmentBytes(&value, length: MemoryLayout<simd_float1>.size, index: 1)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return self.cachedTexture!
}
}
final class EnhanceRenderPass: RenderPass {
private let lightnessPass = EnhanceLightnessPass()
private let lutGeneratorPass = EnhanceLUTGeneratorPass()
private let lookupPass = EnhanceLookupPass()
var value: simd_float1 = 0.0
let clipLimit: Float = 1.25
let tileGridSize: TextureSize = TextureSize(width: 4, height: 4)
func setup(device: MTLDevice, library: MTLLibrary) {
self.lightnessPass.setup(device: device, library: library)
self.lutGeneratorPass.setup(gridSize: self.tileGridSize, device: device, library: library)
self.lookupPass.setup(device: device, library: library)
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
guard self.value > 0.005 else {
return input
}
let dY = (self.tileGridSize.height - (input.height % self.tileGridSize.height)) % self.tileGridSize.height
let dX = (self.tileGridSize.width - (input.width % self.tileGridSize.width)) % self.tileGridSize.width
let lightnessSize = TextureSize(width: input.width + dX, height: input.height + dY)
let lightnessScale = simd_float2(Float(input.width + dX) / Float(input.width), Float(input.height + dY) / Float(input.height))
let lightness = self.lightnessPass.process(input: input, size: lightnessSize, scale: lightnessScale, device: device, commandBuffer: commandBuffer)
let lookupTexture = self.lutGeneratorPass.process(input: lightness!, gridSize: self.tileGridSize, clipLimit: self.clipLimit, device: device, commandBuffer: commandBuffer)
let gridSize = simd_float2(Float(self.tileGridSize.width), Float(self.tileGridSize.height))
let output = self.lookupPass.process(input: input, lookupTexture: lookupTexture!, value: self.value, gridSize: gridSize, device: device, commandBuffer: commandBuffer)
return output
}
}
@@ -0,0 +1,100 @@
import Foundation
import Metal
import simd
import MetalPerformanceShaders
final class HistogramCalculationPass: DefaultRenderPass {
fileprivate var cachedTexture: MTLTexture?
fileprivate var histogramBuffer: MTLBuffer?
fileprivate var calculation: MPSImageHistogram?
var isEnabled = false
var updated: ((Data) -> Void)?
override var fragmentShaderFunctionName: String {
return "histogramPrepareFragmentShader"
}
override var pixelFormat: MTLPixelFormat {
return .r8Unorm
}
override func setup(device: MTLDevice, library: MTLLibrary) {
var histogramInfo = MPSImageHistogramInfo(
numberOfHistogramEntries: 256,
histogramForAlpha: false,
minPixelValue: vector_float4(0,0,0,0),
maxPixelValue: vector_float4(1,1,1,1)
)
let calculation = MPSImageHistogram(device: device, histogramInfo: &histogramInfo)
calculation.zeroHistogram = true
let histogramBufferLength = calculation.histogramSize(forSourceFormat: .bgra8Unorm)
let lumaHistogramBufferLength = calculation.histogramSize(forSourceFormat: .r8Unorm)
if let histogramBuffer = device.makeBuffer(length: histogramBufferLength + lumaHistogramBufferLength, options: [.storageModeShared]) {
self.calculation = calculation
self.histogramBuffer = histogramBuffer
}
super.setup(device: device, library: library)
}
override func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
if self.isEnabled {
self.setupVerticesBuffer(device: device)
let width = input.width
let height = input.height
if self.cachedTexture == nil || self.cachedTexture?.width != width || self.cachedTexture?.height != height {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = width
textureDescriptor.height = height
textureDescriptor.pixelFormat = .r8Unorm
textureDescriptor.storageMode = .shared
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return input
}
self.cachedTexture = texture
texture.label = "lumaTexture"
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(width), height: Double(height),
znear: -1.0, zfar: 1.0)
)
renderCommandEncoder.setFragmentTexture(input, index: 0)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
if let histogramBuffer = self.histogramBuffer, let calculation = self.calculation {
calculation.encode(to: commandBuffer, sourceTexture: input, histogram: histogramBuffer, histogramOffset: 0)
let lumaHistogramBufferLength = calculation.histogramSize(forSourceFormat: .r8Unorm)
calculation.encode(to: commandBuffer, sourceTexture: self.cachedTexture!, histogram: histogramBuffer, histogramOffset: histogramBuffer.length - lumaHistogramBufferLength)
let histogramData = Data(bytes: histogramBuffer.contents(), count: histogramBuffer.length)
self.updated?(histogramData)
}
}
return input
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,410 @@
import Foundation
import AVFoundation
import UIKit
import CoreImage
import Metal
import MetalKit
import Display
import SwiftSignalKit
import TelegramCore
import Postbox
public func mediaEditorGenerateGradientImage(size: CGSize, colors: [UIColor]) -> UIImage? {
UIGraphicsBeginImageContextWithOptions(size, false, 1.0)
if let context = UIGraphicsGetCurrentContext() {
let gradientColors = colors.map { $0.cgColor } as CFArray
let colorSpace = CGColorSpaceCreateDeviceRGB()
var locations: [CGFloat] = [0.0, 1.0]
let gradient = CGGradient(colorsSpace: colorSpace, colors: gradientColors, locations: &locations)!
context.drawLinearGradient(gradient, start: CGPoint(x: 0.0, y: 0.0), end: CGPoint(x: 0.0, y: size.height), options: CGGradientDrawingOptions())
}
let image = UIGraphicsGetImageFromCurrentImageContext()!
UIGraphicsEndImageContext()
return image
}
public func mediaEditorGetGradientColors(from image: UIImage) -> MediaEditor.GradientColors {
let context = DrawingContext(size: CGSize(width: 5.0, height: 5.0), scale: 1.0, clear: false)!
context.withFlippedContext({ context in
if let cgImage = image.cgImage {
context.draw(cgImage, in: CGRect(x: 0.0, y: 0.0, width: 5.0, height: 5.0))
}
})
return MediaEditor.GradientColors(
top: context.colorAt(CGPoint(x: 2.0, y: 0.0)),
bottom: context.colorAt(CGPoint(x: 2.0, y: 4.0))
)
}
private func roundedCornersMaskImage(size: CGSize) -> CIImage {
let image = generateImage(size, opaque: true, scale: 1.0) { size, context in
context.setFillColor(UIColor.black.cgColor)
context.fill(CGRect(origin: .zero, size: size))
context.addPath(CGPath(roundedRect: CGRect(origin: .zero, size: size), cornerWidth: size.width / 8.0, cornerHeight: size.height / 8.0, transform: nil))
context.setFillColor(UIColor.white.cgColor)
context.fillPath()
}?.cgImage
return CIImage(cgImage: image!)
}
private func rectangleMaskImage(size: CGSize) -> CIImage {
let image = generateImage(size, opaque: true, scale: 1.0) { size, context in
context.setFillColor(UIColor.white.cgColor)
context.fill(CGRect(origin: .zero, size: size))
}?.cgImage
return CIImage(cgImage: image!)
}
public final class MediaEditorComposer {
public enum Input {
case texture(MTLTexture, CMTime, Bool, CGRect?, CGFloat, CGPoint)
case videoBuffer(VideoPixelBuffer, CGRect?, CGFloat, CGPoint)
case ciImage(CIImage, CMTime)
var timestamp: CMTime {
switch self {
case let .texture(_, timestamp, _, _, _, _):
return timestamp
case let .videoBuffer(videoBuffer, _, _, _):
return videoBuffer.timestamp
case let .ciImage(_, timestamp):
return timestamp
}
}
var rendererInput: MediaEditorRenderer.Input {
switch self {
case let .texture(texture, timestamp, hasTransparency, rect, scale, offset):
return .texture(texture, timestamp, hasTransparency, rect, scale, offset)
case let .videoBuffer(videoBuffer, rect, scale, offset):
return .videoBuffer(videoBuffer, rect, scale, offset)
case let .ciImage(image, timestamp):
return .ciImage(image, timestamp)
}
}
}
let device: MTLDevice?
let colorSpace: CGColorSpace
let ciContext: CIContext?
private var textureCache: CVMetalTextureCache?
public var values: MediaEditorValues {
didSet {
self.renderChain.update(values: self.values)
self.renderer.videoFinishPass.update(values: self.values, videoDuration: nil, additionalVideoDuration: nil)
}
}
private let dimensions: CGSize
private let outputDimensions: CGSize
private let textScale: CGFloat
private let outputsYuvBuffers: Bool
private let yuvPixelFormat: OSType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
private let renderer = MediaEditorRenderer()
private let renderChain = MediaEditorRenderChain()
private let drawingImage: CIImage?
private var entities: [MediaEditorComposerEntity]
private var maskImage: CIImage?
public init(
postbox: Postbox?,
values: MediaEditorValues,
dimensions: CGSize,
outputDimensions: CGSize,
textScale: CGFloat,
videoDuration: Double?,
additionalVideoDuration: Double?,
outputsYuvBuffers: Bool = false
) {
self.values = values
self.dimensions = dimensions
self.outputDimensions = outputDimensions
self.textScale = textScale
self.outputsYuvBuffers = outputsYuvBuffers
let colorSpace = CGColorSpaceCreateDeviceRGB()
self.colorSpace = colorSpace
self.renderer.addRenderChain(self.renderChain)
if values.isSticker {
self.maskImage = roundedCornersMaskImage(size: CGSize(width: floor(1080.0 * 0.97), height: floor(1080.0 * 0.97)))
} else if values.isAvatar {
self.maskImage = rectangleMaskImage(size: CGSize(width: 1080.0, height: 1080.0))
}
if let drawing = values.drawing, let drawingImage = CIImage(image: drawing, options: [.colorSpace: self.colorSpace]) {
self.drawingImage = drawingImage.transformed(by: CGAffineTransform(translationX: -dimensions.width / 2.0, y: -dimensions.height / 2.0))
} else {
self.drawingImage = nil
}
var entities: [MediaEditorComposerEntity] = []
if let postbox {
for entity in values.entities {
entities.append(contentsOf: composerEntitiesForDrawingEntity(postbox: postbox, textScale: textScale, entity: entity.entity, colorSpace: colorSpace))
}
}
self.entities = entities
self.device = MTLCreateSystemDefaultDevice()
if let device = self.device {
self.ciContext = CIContext(mtlDevice: device, options: [.workingColorSpace : self.colorSpace])
CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &self.textureCache)
} else {
self.ciContext = nil
}
if let maskDrawing = values.maskDrawing, let device = self.device, let maskTexture = loadTexture(image: maskDrawing, device: device) {
self.renderer.currentMainInputMask = maskTexture
}
self.renderer.setupForComposer(composer: self)
self.renderChain.update(values: self.values)
self.renderer.videoFinishPass.update(values: self.values, videoDuration: videoDuration, additionalVideoDuration: additionalVideoDuration)
}
var previousAdditionalInput: [Int: Input] = [:]
public func process(main: Input, additional: [Input?], timestamp: CMTime, pool: CVPixelBufferPool?, completion: @escaping (CVPixelBuffer?) -> Void) {
guard let pool, let ciContext = self.ciContext else {
completion(nil)
return
}
var index = 0
var augmentedAdditionals: [Input?] = []
for input in additional {
if let input {
self.previousAdditionalInput[index] = input
augmentedAdditionals.append(input)
} else {
augmentedAdditionals.append(self.previousAdditionalInput[index])
}
index += 1
}
self.renderer.consume(main: main.rendererInput, additionals: augmentedAdditionals.compactMap { $0 }.map { $0.rendererInput }, render: true)
if let resultTexture = self.renderer.resultTexture, var ciImage = CIImage(mtlTexture: resultTexture, options: [.colorSpace: self.colorSpace]) {
ciImage = ciImage.transformed(by: CGAffineTransformMakeScale(1.0, -1.0).translatedBy(x: 0.0, y: -ciImage.extent.height))
var pixelBuffer: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixelBuffer)
guard let pixelBuffer else {
completion(nil)
return
}
if self.outputsYuvBuffers {
let scale = self.outputDimensions.width / ciImage.extent.width
ciImage = ciImage.samplingLinear().transformed(by: CGAffineTransform(scaleX: scale, y: scale))
ciContext.render(ciImage, to: pixelBuffer)
completion(pixelBuffer)
} else {
makeEditorImageFrameComposition(context: ciContext, inputImage: ciImage, drawingImage: self.drawingImage, maskImage: self.maskImage, dimensions: self.dimensions, values: self.values, entities: self.entities, time: timestamp, completion: { compositedImage in
if var compositedImage {
let scale = self.outputDimensions.width / compositedImage.extent.width
compositedImage = compositedImage.samplingLinear().transformed(by: CGAffineTransform(scaleX: scale, y: scale))
ciContext.render(compositedImage, to: pixelBuffer)
completion(pixelBuffer)
} else {
completion(nil)
}
})
}
}
}
private var cachedTextures: [Int: MTLTexture] = [:]
func textureForImage(index: Int, image: UIImage) -> MTLTexture? {
if let cachedTexture = self.cachedTextures[index] {
return cachedTexture
}
if let device = self.device, let texture = loadTexture(image: image, device: device) {
self.cachedTextures[index] = texture
return texture
}
return nil
}
}
public func makeEditorImageComposition(context: CIContext, postbox: Postbox, inputImage: UIImage, dimensions: CGSize, outputDimensions: CGSize? = nil, values: MediaEditorValues, time: CMTime, textScale: CGFloat, completion: @escaping (UIImage?) -> Void) {
let colorSpace = CGColorSpaceCreateDeviceRGB()
let inputImage = CIImage(image: inputImage, options: [.colorSpace: colorSpace])!
var drawingImage: CIImage?
var maskImage: CIImage?
if values.isSticker {
maskImage = roundedCornersMaskImage(size: CGSize(width: floor(1080.0 * 0.97), height: floor(1080.0 * 0.97)))
} else if values.isAvatar {
maskImage = rectangleMaskImage(size: CGSize(width: 1080.0, height: 1080.0))
} else if let outputDimensions {
maskImage = rectangleMaskImage(size: outputDimensions.aspectFitted(CGSize(width: 1080.0, height: 1920.0)))
}
if let drawing = values.drawing, let image = CIImage(image: drawing, options: [.colorSpace: colorSpace]) {
drawingImage = image.transformed(by: CGAffineTransform(translationX: -dimensions.width / 2.0, y: -dimensions.height / 2.0))
}
var entities: [MediaEditorComposerEntity] = []
for entity in values.entities {
entities.append(contentsOf: composerEntitiesForDrawingEntity(postbox: postbox, textScale: textScale, entity: entity.entity, colorSpace: colorSpace))
}
makeEditorImageFrameComposition(context: context, inputImage: inputImage, drawingImage: drawingImage, maskImage: maskImage, dimensions: dimensions, outputDimensions: outputDimensions, values: values, entities: entities, time: time, textScale: textScale, completion: { compositedImage in
if var compositedImage {
let outputDimensions = outputDimensions ?? dimensions
let scale = outputDimensions.width / compositedImage.extent.width
compositedImage = compositedImage.samplingLinear().transformed(by: CGAffineTransform(scaleX: scale, y: scale))
if let cgImage = context.createCGImage(compositedImage, from: CGRect(origin: .zero, size: compositedImage.extent.size)) {
Queue.mainQueue().async {
completion(UIImage(cgImage: cgImage))
}
return
}
}
Queue.mainQueue().async {
completion(nil)
}
})
}
private func makeEditorImageFrameComposition(context: CIContext, inputImage: CIImage, drawingImage: CIImage?, maskImage: CIImage?, dimensions: CGSize, outputDimensions: CGSize? = nil, values: MediaEditorValues, entities: [MediaEditorComposerEntity], time: CMTime, textScale: CGFloat = 1.0, completion: @escaping (CIImage?) -> Void) {
var isClear = false
if let gradientColor = values.gradientColors?.first, gradientColor.alpha.isZero {
isClear = true
}
var resultImage = CIImage(color: isClear ? .clear : .black).cropped(to: CGRect(origin: .zero, size: dimensions)).transformed(by: CGAffineTransform(translationX: -dimensions.width / 2.0, y: -dimensions.height / 2.0))
var mediaImage = inputImage.samplingLinear().transformed(by: CGAffineTransform(translationX: -inputImage.extent.midX, y: -inputImage.extent.midY))
if values.isStory || values.isSticker || values.isAvatar || values.isCover {
resultImage = mediaImage.samplingLinear().composited(over: resultImage)
} else {
let initialScale = dimensions.width / mediaImage.extent.width
var horizontalScale = initialScale
if values.cropMirroring {
horizontalScale *= -1.0
}
mediaImage = mediaImage.transformed(by: CGAffineTransformMakeScale(horizontalScale, initialScale))
resultImage = mediaImage.composited(over: resultImage)
}
if let drawingImage {
resultImage = drawingImage.samplingLinear().composited(over: resultImage)
}
let frameRate: Float = 30.0
let entitiesCount = Atomic<Int>(value: 1)
let entitiesImages = Atomic<[(CIImage, Int)]>(value: [])
let maybeFinalize = {
let count = entitiesCount.modify { current -> Int in
return current - 1
}
if count == 0 {
let sortedImages = entitiesImages.with({ $0 }).sorted(by: { $0.1 < $1.1 }).map({ $0.0 })
for image in sortedImages {
resultImage = image.composited(over: resultImage)
}
resultImage = resultImage.transformed(by: CGAffineTransform(translationX: dimensions.width / 2.0, y: dimensions.height / 2.0))
if values.isSticker {
let minSize = min(dimensions.width, dimensions.height)
let scaledSize = CGSize(width: floor(minSize * 0.97), height: floor(minSize * 0.97))
resultImage = resultImage.transformed(by: CGAffineTransform(translationX: -(dimensions.width - scaledSize.width) / 2.0, y: -(dimensions.height - scaledSize.height) / 2.0)).cropped(to: CGRect(origin: .zero, size: scaledSize))
} else if values.isAvatar {
let minSize = min(dimensions.width, dimensions.height)
let scaledSize = CGSize(width: minSize, height: minSize)
resultImage = resultImage.transformed(by: CGAffineTransform(translationX: -(dimensions.width - scaledSize.width) / 2.0, y: -(dimensions.height - scaledSize.height) / 2.0)).cropped(to: CGRect(origin: .zero, size: scaledSize))
} else if values.isCover, let outputDimensions {
let scaledSize = outputDimensions.aspectFitted(dimensions)
resultImage = resultImage.transformed(by: CGAffineTransform(translationX: -(dimensions.width - scaledSize.width) / 2.0, y: -(dimensions.height - scaledSize.height) / 2.0)).cropped(to: CGRect(origin: .zero, size: scaledSize))
} else if values.isStory {
resultImage = resultImage.cropped(to: CGRect(origin: .zero, size: dimensions))
} else {
let originalDimensions = values.originalDimensions.cgSize
var cropRect = values.cropRect ?? .zero
if cropRect.isEmpty {
cropRect = CGRect(origin: .zero, size: originalDimensions)
}
let scale = dimensions.width / originalDimensions.width
let scaledCropRect = CGRect(origin: CGPoint(x: cropRect.minX * scale, y: dimensions.height - cropRect.maxY * scale), size: CGSize(width: cropRect.width * scale, height: cropRect.height * scale))
resultImage = resultImage.cropped(to: scaledCropRect)
resultImage = resultImage.transformed(by: CGAffineTransformMakeTranslation(-scaledCropRect.minX, -scaledCropRect.minY))
if let orientation = values.cropOrientation, orientation != .up {
let rotation = orientation.rotation
resultImage = resultImage.transformed(by: CGAffineTransformMakeTranslation(-resultImage.extent.width / 2.0, -resultImage.extent.height / 2.0))
resultImage = resultImage.transformed(by: CGAffineTransformMakeRotation(rotation))
resultImage = resultImage.transformed(by: CGAffineTransformMakeTranslation(resultImage.extent.width / 2.0, resultImage.extent.height / 2.0))
}
}
if let maskImage, let filter = CIFilter(name: "CIBlendWithMask") {
filter.setValue(resultImage, forKey: kCIInputImageKey)
filter.setValue(maskImage, forKey: kCIInputMaskImageKey)
filter.setValue(CIImage(color: .clear), forKey: kCIInputBackgroundImageKey)
if let blendedImage = filter.outputImage {
completion(blendedImage.cropped(to: resultImage.extent))
} else {
completion(resultImage)
}
} else {
completion(resultImage)
}
}
}
var i = 0
for entity in entities {
let _ = entitiesCount.modify { current -> Int in
return current + 1
}
let index = i
entity.image(for: time, frameRate: frameRate, context: context, completion: { image in
if var image = image?.samplingLinear() {
let resetTransform = CGAffineTransform(translationX: -image.extent.width / 2.0, y: -image.extent.height / 2.0)
image = image.transformed(by: resetTransform)
var baseScale: CGFloat = 1.0
if let scale = entity.baseScale {
baseScale = scale
} else if let _ = entity.baseDrawingSize {
} else if let baseSize = entity.baseSize {
baseScale = baseSize.width / image.extent.width
}
var transform = CGAffineTransform.identity
transform = transform.translatedBy(x: -dimensions.width / 2.0 + entity.position.x, y: dimensions.height / 2.0 + entity.position.y * -1.0)
transform = transform.rotated(by: -entity.rotation)
transform = transform.scaledBy(x: entity.scale * baseScale, y: entity.scale * baseScale)
if entity.mirrored {
transform = transform.scaledBy(x: -1.0, y: 1.0)
}
image = image.transformed(by: transform)
let _ = entitiesImages.modify { current in
var updated = current
updated.append((image, index))
return updated
}
}
maybeFinalize()
})
i += 1
}
maybeFinalize()
}
@@ -0,0 +1,704 @@
import Foundation
import AVFoundation
import UIKit
import CoreImage
import Metal
import MetalKit
import Display
import SwiftSignalKit
import TelegramCore
import Postbox
import AnimatedStickerNode
import TelegramAnimatedStickerNode
import YuvConversion
import StickerResources
private func prerenderTextTransformations(entity: DrawingEntity, image: UIImage, textScale: CGFloat, colorSpace: CGColorSpace) -> MediaEditorComposerStaticEntity {
let imageSize = image.size
let angle: CGFloat
var scale: CGFloat
let position: CGPoint
if let entity = entity as? DrawingTextEntity {
angle = -entity.rotation
scale = entity.scale
position = entity.position
} else if let entity = entity as? DrawingLocationEntity {
angle = -entity.rotation
scale = entity.scale
position = entity.position
} else if let entity = entity as? DrawingWeatherEntity {
angle = -entity.rotation
scale = entity.scale
position = entity.position
} else if let entity = entity as? DrawingLinkEntity {
angle = -entity.rotation
scale = entity.scale
position = entity.position
} else {
fatalError()
}
scale *= 0.5 * textScale
let rotatedSize = CGSize(
width: abs(imageSize.width * cos(angle)) + abs(imageSize.height * sin(angle)),
height: abs(imageSize.width * sin(angle)) + abs(imageSize.height * cos(angle))
)
let newSize = CGSize(width: rotatedSize.width * scale, height: rotatedSize.height * scale)
let newImage = generateImage(newSize, contextGenerator: { size, context in
context.setAllowsAntialiasing(true)
context.setShouldAntialias(true)
context.interpolationQuality = .high
context.clear(CGRect(origin: .zero, size: size))
context.translateBy(x: newSize.width * 0.5, y: newSize.height * 0.5)
context.rotate(by: angle)
context.scaleBy(x: scale, y: scale)
let drawRect = CGRect(
x: -imageSize.width * 0.5,
y: -imageSize.height * 0.5,
width: imageSize.width,
height: imageSize.height
)
if let cgImage = image.cgImage {
context.draw(cgImage, in: drawRect)
}
}, scale: 1.0)!
return MediaEditorComposerStaticEntity(image: CIImage(image: newImage, options: [.colorSpace: colorSpace])!, position: position, scale: 1.0, rotation: 0.0, baseSize: nil, baseDrawingSize: CGSize(width: 1080, height: 1920), mirrored: false)
}
func composerEntitiesForDrawingEntity(postbox: Postbox, textScale: CGFloat, entity: DrawingEntity, colorSpace: CGColorSpace, tintColor: UIColor? = nil) -> [MediaEditorComposerEntity] {
if entity is DrawingWeatherEntity {
return []
} else if let entity = entity as? DrawingStickerEntity {
if case let .file(_, type) = entity.content, case .reaction = type {
return []
} else {
let content: MediaEditorComposerStickerEntity.Content
var scale = entity.scale
switch entity.content {
case let .file(file, _):
content = .file(file.media)
case let .image(image, _):
content = .image(image)
case let .animatedImage(data, _):
if let animatedImage = UIImage.animatedImageFromData(data: data) {
content = .animatedImage(animatedImage.images, animatedImage.duration)
scale *= 1.0
} else {
return []
}
case let .video(file):
content = .video(file)
case .dualVideoReference:
return []
case .message, .gift:
if let renderImage = entity.renderImage, let image = CIImage(image: renderImage, options: [.colorSpace: colorSpace]) {
var entities: [MediaEditorComposerEntity] = []
entities.append(MediaEditorComposerStaticEntity(image: image, position: entity.position, scale: entity.scale, rotation: entity.rotation, baseSize: entity.baseSize, mirrored: false))
if let renderSubEntities = entity.renderSubEntities {
for subEntity in renderSubEntities {
entities.append(contentsOf: composerEntitiesForDrawingEntity(postbox: postbox, textScale: textScale, entity: subEntity, colorSpace: colorSpace))
}
}
return entities
} else {
return []
}
}
return [MediaEditorComposerStickerEntity(postbox: postbox, content: content, position: entity.position, scale: scale, rotation: entity.rotation, baseSize: entity.baseSize, mirrored: entity.mirrored, colorSpace: colorSpace, tintColor: tintColor, isStatic: entity.isExplicitlyStatic)]
}
} else if let renderImage = entity.renderImage, let image = CIImage(image: renderImage, options: [.colorSpace: colorSpace]) {
if let entity = entity as? DrawingBubbleEntity {
return [MediaEditorComposerStaticEntity(image: image, position: entity.position, scale: 1.0, rotation: entity.rotation, baseSize: entity.size, mirrored: false)]
} else if let entity = entity as? DrawingSimpleShapeEntity {
return [MediaEditorComposerStaticEntity(image: image, position: entity.position, scale: 1.0, rotation: entity.rotation, baseSize: entity.size, mirrored: false)]
} else if let entity = entity as? DrawingVectorEntity {
return [MediaEditorComposerStaticEntity(image: image, position: CGPoint(x: entity.drawingSize.width * 0.5, y: entity.drawingSize.height * 0.5), scale: 1.0, rotation: 0.0, baseSize: entity.drawingSize, mirrored: false)]
} else if let entity = entity as? DrawingTextEntity {
var entities: [MediaEditorComposerEntity] = []
entities.append(prerenderTextTransformations(entity: entity, image: renderImage, textScale: textScale, colorSpace: colorSpace))
if let renderSubEntities = entity.renderSubEntities {
for subEntity in renderSubEntities {
entities.append(contentsOf: composerEntitiesForDrawingEntity(postbox: postbox, textScale: textScale, entity: subEntity, colorSpace: colorSpace, tintColor: entity.color.toUIColor()))
}
}
return entities
} else if let entity = entity as? DrawingLocationEntity {
return [prerenderTextTransformations(entity: entity, image: renderImage, textScale: textScale, colorSpace: colorSpace)]
} else if let entity = entity as? DrawingLinkEntity {
return [prerenderTextTransformations(entity: entity, image: renderImage, textScale: textScale, colorSpace: colorSpace)]
} else if let entity = entity as? DrawingWeatherEntity {
return [prerenderTextTransformations(entity: entity, image: renderImage, textScale: textScale, colorSpace: colorSpace)]
}
}
return []
}
private class MediaEditorComposerStaticEntity: MediaEditorComposerEntity {
let image: CIImage
let position: CGPoint
let scale: CGFloat
let rotation: CGFloat
let baseSize: CGSize?
let baseScale: CGFloat?
let baseDrawingSize: CGSize?
let mirrored: Bool
init(
image: CIImage,
position: CGPoint,
scale: CGFloat,
rotation: CGFloat,
baseSize: CGSize?,
baseScale: CGFloat? = nil,
baseDrawingSize: CGSize? = nil,
mirrored: Bool
) {
self.image = image
self.position = position
self.scale = scale
self.rotation = rotation
self.baseSize = baseSize
self.baseScale = baseScale
self.baseDrawingSize = baseDrawingSize
self.mirrored = mirrored
}
func image(for time: CMTime, frameRate: Float, context: CIContext, completion: @escaping (CIImage?) -> Void) {
completion(self.image)
}
}
final class MediaEditorComposerStickerEntity: MediaEditorComposerEntity {
public enum Content {
case file(TelegramMediaFile)
case video(TelegramMediaFile)
case image(UIImage)
case animatedImage([UIImage], Double)
var file: TelegramMediaFile? {
if case let .file(file) = self {
return file
}
return nil
}
}
let postbox: Postbox
let content: Content
let position: CGPoint
let scale: CGFloat
let rotation: CGFloat
let baseSize: CGSize?
let baseScale: CGFloat? = nil
let baseDrawingSize: CGSize? = nil
let mirrored: Bool
let colorSpace: CGColorSpace
let tintColor: UIColor?
let isStatic: Bool
var isAnimated: Bool
var source: AnimatedStickerNodeSource?
var frameSource = Promise<QueueLocalObject<AnimatedStickerDirectFrameSource>?>()
var videoFrameSource = Promise<QueueLocalObject<VideoStickerDirectFrameSource>?>()
var isVideoSticker = false
var assetReader: AVAssetReader?
var videoOutput: AVAssetReaderTrackOutput?
var frameCount: Int?
var frameRate: Int?
var currentFrameIndex: Int?
var totalDuration: Double?
let durationPromise = Promise<Double>()
let queue = Queue()
let disposables = DisposableSet()
var image: CIImage?
var imagePixelBuffer: CVPixelBuffer?
let imagePromise = Promise<UIImage>()
init(postbox: Postbox, content: Content, position: CGPoint, scale: CGFloat, rotation: CGFloat, baseSize: CGSize, mirrored: Bool, colorSpace: CGColorSpace, tintColor: UIColor?, isStatic: Bool, highRes: Bool = false) {
self.postbox = postbox
self.content = content
self.position = position
self.scale = scale
self.rotation = rotation
self.baseSize = baseSize
self.mirrored = mirrored
self.colorSpace = colorSpace
self.tintColor = tintColor
self.isStatic = isStatic
switch content {
case let .file(file):
let dimensions = file.dimensions ?? PixelDimensions(width: 512, height: 512)
if file.isAnimatedSticker || file.isVideoSticker || file.mimeType == "video/webm" {
self.isAnimated = true
self.isVideoSticker = file.isVideoSticker || file.mimeType == "video/webm"
self.source = AnimatedStickerResourceSource(postbox: postbox, resource: file.resource, isVideo: isVideoSticker)
let pathPrefix = postbox.mediaBox.shortLivedResourceCachePathPrefix(file.resource.id)
if let source = self.source {
let fitToSize: CGSize
if highRes {
fitToSize = CGSize(width: 512, height: 512)
} else if self.isStatic {
fitToSize = CGSize(width: 768, height: 768)
} else {
fitToSize = CGSize(width: 384, height: 384)
}
let fittedDimensions = dimensions.cgSize.aspectFitted(fitToSize)
self.disposables.add((source.directDataPath(attemptSynchronously: true)
|> deliverOn(self.queue)).start(next: { [weak self] path in
if let strongSelf = self, let path {
if let data = try? Data(contentsOf: URL(fileURLWithPath: path), options: [.mappedRead]) {
let queue = strongSelf.queue
if strongSelf.isVideoSticker {
let frameSource = QueueLocalObject<VideoStickerDirectFrameSource>(queue: queue, generate: {
return VideoStickerDirectFrameSource(queue: queue, path: path, width: Int(fittedDimensions.width), height: Int(fittedDimensions.height), cachePathPrefix: pathPrefix, unpremultiplyAlpha: false)!
})
frameSource.syncWith { frameSource in
strongSelf.frameCount = frameSource.frameCount
strongSelf.frameRate = frameSource.frameRate
let duration: Double
if frameSource.frameCount > 0 {
duration = Double(frameSource.frameCount) / Double(frameSource.frameRate)
} else {
duration = frameSource.duration
}
strongSelf.totalDuration = duration
strongSelf.durationPromise.set(.single(duration))
}
strongSelf.videoFrameSource.set(.single(frameSource))
} else {
let frameSource = QueueLocalObject<AnimatedStickerDirectFrameSource>(queue: queue, generate: {
return AnimatedStickerDirectFrameSource(queue: queue, data: data, width: Int(fittedDimensions.width), height: Int(fittedDimensions.height), cachePathPrefix: pathPrefix, useMetalCache: false, fitzModifier: nil)!
})
frameSource.syncWith { frameSource in
strongSelf.frameCount = frameSource.frameCount
strongSelf.frameRate = frameSource.frameRate
let duration = Double(frameSource.frameCount) / Double(frameSource.frameRate)
strongSelf.totalDuration = duration
strongSelf.durationPromise.set(.single(duration))
}
strongSelf.frameSource.set(.single(frameSource))
}
}
}
}))
}
} else {
self.isAnimated = false
self.disposables.add((chatMessageSticker(postbox: postbox, userLocation: .other, file: file, small: false, fetched: true, onlyFullSize: true, thumbnail: false, synchronousLoad: false, colorSpace: self.colorSpace)
|> deliverOn(self.queue)).start(next: { [weak self] generator in
if let self {
let context = generator(TransformImageArguments(corners: ImageCorners(), imageSize: baseSize, boundingSize: baseSize, intrinsicInsets: UIEdgeInsets()))
let image = context?.generateImage(colorSpace: self.colorSpace)
if let image {
self.imagePromise.set(.single(image))
}
}
}))
}
case let .image(image):
self.isAnimated = false
self.imagePromise.set(.single(image))
case let .animatedImage(images, duration):
self.isAnimated = true
self.videoFrameRate = Float(images.count) / Float(duration)
self.totalDuration = duration
self.durationPromise.set(.single(duration))
case .video:
self.isAnimated = true
}
}
deinit {
self.disposables.dispose()
}
private func setupVideoOutput() {
if case let .video(file) = self.content {
if let path = self.postbox.mediaBox.completedResourcePath(file.resource, pathExtension: "mp4") {
let url = URL(fileURLWithPath: path)
let asset = AVURLAsset(url: url)
if let assetReader = try? AVAssetReader(asset: asset), let videoTrack = asset.tracks(withMediaType: .video).first {
self.videoFrameRate = videoTrack.nominalFrameRate
let outputSettings: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
kCVPixelBufferMetalCompatibilityKey as String: true
]
let videoOutput = AVAssetReaderTrackOutput(track: videoTrack, outputSettings: outputSettings)
videoOutput.alwaysCopiesSampleData = true
if assetReader.canAdd(videoOutput) {
assetReader.add(videoOutput)
}
assetReader.startReading()
self.assetReader = assetReader
self.videoOutput = videoOutput
}
}
}
}
private var videoFrameRate: Float?
private var maskFilter: CIFilter?
func image(for time: CMTime, frameRate: Float, context: CIContext, completion: @escaping (CIImage?) -> Void) {
let currentTime = CMTimeGetSeconds(time)
if case let .animatedImage(images, _) = self.content {
var frameAdvancement: Int = 0
if let frameRate = self.videoFrameRate, frameRate > 0 {
let frameTime = 1.0 / Double(frameRate)
let frameIndex = Int(floor(currentTime / frameTime))
let currentFrameIndex = self.currentFrameIndex
if currentFrameIndex != frameIndex {
let previousFrameIndex = currentFrameIndex
self.currentFrameIndex = frameIndex
var delta = 1
if let previousFrameIndex = previousFrameIndex {
delta = max(1, frameIndex - previousFrameIndex)
}
frameAdvancement = delta
}
}
if frameAdvancement == 0, let image = self.image {
completion(image)
return
} else if let currentFrameIndex = self.currentFrameIndex {
let index = currentFrameIndex % images.count
var image = images[index]
image = generateScaledImage(image: images[index], size: image.size.aspectFitted(CGSize(width: 384, height: 384)), opaque: false, scale: 1.0)!
let ciImage = CIImage(image: image)
self.image = ciImage
completion(ciImage)
return
}
} else if case .video = self.content {
if self.videoOutput == nil {
self.setupVideoOutput()
}
if let videoOutput = self.videoOutput {
var frameAdvancement: Int = 0
if let frameRate = self.videoFrameRate, frameRate > 0 {
let frameTime = 1.0 / Double(frameRate)
let frameIndex = Int(floor(currentTime / frameTime))
let currentFrameIndex = self.currentFrameIndex
if currentFrameIndex != frameIndex {
let previousFrameIndex = currentFrameIndex
self.currentFrameIndex = frameIndex
var delta = 1
if let previousFrameIndex = previousFrameIndex {
delta = max(1, frameIndex - previousFrameIndex)
}
frameAdvancement = delta
}
}
if frameAdvancement == 0, let image = self.image {
completion(image)
return
} else {
var sampleBuffer = videoOutput.copyNextSampleBuffer()
if sampleBuffer == nil && self.assetReader?.status == .completed {
self.setupVideoOutput()
sampleBuffer = self.videoOutput?.copyNextSampleBuffer()
}
if let sampleBuffer, let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
var ciImage = CIImage(cvPixelBuffer: imageBuffer)
var circleMaskFilter: CIFilter?
if let current = self.maskFilter {
circleMaskFilter = current
} else {
let circleImage = generateImage(ciImage.extent.size, scale: 1.0, rotatedContext: { size, context in
context.clear(CGRect(origin: .zero, size: size))
context.setFillColor(UIColor.white.cgColor)
let path = UIBezierPath(roundedRect: CGRect(origin: .zero, size: size), cornerRadius: floor(size.width * 0.03))
context.addPath(path.cgPath)
context.fillPath()
})!
let circleMask = CIImage(image: circleImage)
if let filter = CIFilter(name: "CIBlendWithAlphaMask") {
filter.setValue(circleMask, forKey: kCIInputMaskImageKey)
self.maskFilter = filter
circleMaskFilter = filter
}
}
let _ = circleMaskFilter
if let circleMaskFilter {
circleMaskFilter.setValue(ciImage, forKey: kCIInputImageKey)
if let output = circleMaskFilter.outputImage {
ciImage = output
}
}
self.image = ciImage
completion(ciImage)
}
}
} else {
completion(nil)
}
} else if self.isAnimated {
var tintColor: UIColor?
if let file = self.content.file, file.isCustomTemplateEmoji {
tintColor = self.tintColor ?? UIColor(rgb: 0xffffff)
}
let processFrame: (Double?, Int?, Int?, (Int) -> AnimatedStickerFrame?) -> Void = { [weak self] duration, frameCount, frameRate, takeFrame in
guard let strongSelf = self else {
completion(nil)
return
}
var frameAdvancement: Int = 0
if let duration, let frameCount, frameCount > 0 {
let relativeTime = currentTime - floor(currentTime / duration) * duration
var t = relativeTime / duration
t = max(0.0, t)
t = min(1.0, t)
let startFrame: Double = 0
let endFrame = Double(frameCount)
let frameOffset = Int(Double(startFrame) * (1.0 - t) + Double(endFrame - 1) * t)
let lowerBound: Int = 0
let upperBound = frameCount - 1
let frameIndex = max(lowerBound, min(upperBound, frameOffset))
let currentFrameIndex = strongSelf.currentFrameIndex
if currentFrameIndex != frameIndex {
let previousFrameIndex = currentFrameIndex
strongSelf.currentFrameIndex = frameIndex
var delta = 1
if let previousFrameIndex = previousFrameIndex {
delta = max(1, frameIndex - previousFrameIndex)
}
frameAdvancement = delta
}
} else if let frameRate, frameRate > 0 {
let frameTime = 1.0 / Double(frameRate)
let frameIndex = Int(floor(currentTime / frameTime))
let currentFrameIndex = strongSelf.currentFrameIndex
if currentFrameIndex != frameIndex {
let previousFrameIndex = currentFrameIndex
strongSelf.currentFrameIndex = frameIndex
var delta = 1
if let previousFrameIndex = previousFrameIndex {
delta = max(1, frameIndex - previousFrameIndex)
}
frameAdvancement = delta
}
}
if frameAdvancement == 0, let image = strongSelf.image {
completion(image)
} else {
if let frame = takeFrame(max(1, frameAdvancement)) {
var imagePixelBuffer: CVPixelBuffer?
if let pixelBuffer = strongSelf.imagePixelBuffer {
imagePixelBuffer = pixelBuffer
} else {
let ioSurfaceProperties = NSMutableDictionary()
let options = NSMutableDictionary()
options.setObject(ioSurfaceProperties, forKey: kCVPixelBufferIOSurfacePropertiesKey as NSString)
var pixelBuffer: CVPixelBuffer?
CVPixelBufferCreate(
kCFAllocatorDefault,
frame.width,
frame.height,
kCVPixelFormatType_32BGRA,
options,
&pixelBuffer
)
imagePixelBuffer = pixelBuffer
strongSelf.imagePixelBuffer = pixelBuffer
}
if let imagePixelBuffer {
let image = render(context: context, width: frame.width, height: frame.height, bytesPerRow: frame.bytesPerRow, data: frame.data, type: frame.type, pixelBuffer: imagePixelBuffer, tintColor: tintColor)
strongSelf.image = image
}
completion(strongSelf.image)
} else {
completion(strongSelf.image)
}
}
}
if self.isVideoSticker {
self.disposables.add((self.videoFrameSource.get()
|> take(1)
|> deliverOn(self.queue)).start(next: { [weak self] frameSource in
guard let strongSelf = self else {
completion(nil)
return
}
guard let frameSource else {
completion(nil)
return
}
processFrame(strongSelf.totalDuration, strongSelf.frameCount, strongSelf.frameRate, { delta in
var frame: AnimatedStickerFrame?
frameSource.syncWith { frameSource in
for i in 0 ..< delta {
frame = frameSource.takeFrame(draw: i == delta - 1)
}
}
return frame
})
}))
} else {
self.disposables.add((self.frameSource.get()
|> take(1)
|> deliverOn(self.queue)).start(next: { [weak self] frameSource in
guard let strongSelf = self else {
completion(nil)
return
}
guard let frameSource else {
completion(nil)
return
}
processFrame(strongSelf.totalDuration, strongSelf.frameCount, strongSelf.frameRate, { delta in
var frame: AnimatedStickerFrame?
frameSource.syncWith { frameSource in
for i in 0 ..< delta {
frame = frameSource.takeFrame(draw: i == delta - 1)
}
}
return frame
})
}))
}
} else {
var image: CIImage?
if let cachedImage = self.image {
image = cachedImage
completion(image)
} else {
let _ = (self.imagePromise.get()
|> take(1)
|> deliverOn(self.queue)).start(next: { [weak self] image in
if let self {
self.image = CIImage(image: image, options: [.colorSpace: self.colorSpace])
completion(self.image)
}
})
}
}
}
}
protocol MediaEditorComposerEntity {
var position: CGPoint { get }
var scale: CGFloat { get }
var rotation: CGFloat { get }
var baseSize: CGSize? { get }
var baseScale: CGFloat? { get }
var baseDrawingSize: CGSize? { get }
var mirrored: Bool { get }
func image(for time: CMTime, frameRate: Float, context: CIContext, completion: @escaping (CIImage?) -> Void)
}
extension CIImage {
func tinted(with color: UIColor) -> CIImage? {
guard let colorMatrix = CIFilter(name: "CIColorMatrix") else {
return self
}
var r: CGFloat = 0, g: CGFloat = 0, b: CGFloat = 0, a: CGFloat = 0
color.getRed(&r, green: &g, blue: &b, alpha: &a)
colorMatrix.setDefaults()
colorMatrix.setValue(self, forKey: "inputImage")
colorMatrix.setValue(CIVector(x: r, y: 0, z: 0, w: 0), forKey: "inputRVector")
colorMatrix.setValue(CIVector(x: 0, y: g, z: 0, w: 0), forKey: "inputGVector")
colorMatrix.setValue(CIVector(x: 0, y: 0, z: b, w: 0), forKey: "inputBVector")
colorMatrix.setValue(CIVector(x: 0, y: 0, z: 0, w: a), forKey: "inputAVector")
return colorMatrix.outputImage
}
}
private func render(context: CIContext, width: Int, height: Int, bytesPerRow: Int, data: Data, type: AnimationRendererFrameType, pixelBuffer: CVPixelBuffer, tintColor: UIColor?) -> CIImage? {
let calculatedBytesPerRow = (4 * Int(width) + 31) & (~31)
//assert(bytesPerRow == calculatedBytesPerRow)
CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let dest = CVPixelBufferGetBaseAddress(pixelBuffer)
switch type {
case .yuva:
data.withUnsafeBytes { buffer -> Void in
guard let bytes = buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else {
return
}
decodeYUVAToRGBA(bytes, dest, Int32(width), Int32(height), Int32(calculatedBytesPerRow))
}
case .argb:
data.withUnsafeBytes { buffer -> Void in
guard let bytes = buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else {
return
}
memcpy(dest, bytes, data.count)
}
case .dct:
break
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
let image = CIImage(cvPixelBuffer: pixelBuffer, options: [.colorSpace: deviceColorSpace])
if let tintColor {
if let cgImage = context.createCGImage(image, from: CGRect(origin: .zero, size: image.extent.size)) {
if let tintedImage = generateTintedImage(image: UIImage(cgImage: cgImage), color: tintColor) {
return CIImage(image: tintedImage)
}
}
}
return image
}
private extension UIImage.Orientation {
var exifOrientation: Int32 {
switch self {
case .up: return 1
case .down: return 3
case .left: return 8
case .right: return 6
case .upMirrored: return 2
case .downMirrored: return 4
case .leftMirrored: return 5
case .rightMirrored: return 7
@unknown default:
return 0
}
}
}
@@ -0,0 +1,296 @@
import Foundation
import UIKit
import SwiftSignalKit
import CoreLocation
import TelegramCore
import TelegramUIPreferences
import PersistentStringHash
import Postbox
import AccountContext
public struct MediaEditorResultPrivacy: Codable, Equatable {
private enum CodingKeys: String, CodingKey {
case sendAsPeerId
case privacy
case timeout
case disableForwarding
case archive
case folderIds
}
public let sendAsPeerId: EnginePeer.Id?
public let privacy: EngineStoryPrivacy
public let timeout: Int
public let isForwardingDisabled: Bool
public let pin: Bool
public let folderIds: [Int64]
public init(
sendAsPeerId: EnginePeer.Id?,
privacy: EngineStoryPrivacy,
timeout: Int,
isForwardingDisabled: Bool,
pin: Bool,
folderIds: [Int64]
) {
self.sendAsPeerId = sendAsPeerId
self.privacy = privacy
self.timeout = timeout
self.isForwardingDisabled = isForwardingDisabled
self.pin = pin
self.folderIds = folderIds
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.sendAsPeerId = try container.decodeIfPresent(Int64.self, forKey: .sendAsPeerId).flatMap { EnginePeer.Id($0) }
self.privacy = try container.decode(EngineStoryPrivacy.self, forKey: .privacy)
self.timeout = Int(try container.decode(Int32.self, forKey: .timeout))
self.isForwardingDisabled = try container.decodeIfPresent(Bool.self, forKey: .disableForwarding) ?? false
self.pin = try container.decode(Bool.self, forKey: .archive)
self.folderIds = try container.decodeIfPresent([Int64].self, forKey: .folderIds) ?? []
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encodeIfPresent(self.sendAsPeerId?.toInt64(), forKey: .sendAsPeerId)
try container.encode(self.privacy, forKey: .privacy)
try container.encode(Int32(self.timeout), forKey: .timeout)
try container.encode(self.isForwardingDisabled, forKey: .disableForwarding)
try container.encode(self.pin, forKey: .archive)
try container.encode(self.folderIds, forKey: .folderIds)
}
}
public final class MediaEditorDraft: Codable, Equatable {
enum ReadError: Error {
case generic
}
public static func == (lhs: MediaEditorDraft, rhs: MediaEditorDraft) -> Bool {
return lhs.path == rhs.path
}
private enum CodingKeys: String, CodingKey {
case path
case isVideo
case thumbnail
case dimensionsWidth
case dimensionsHeight
case duration
case values
case caption
case privacy
case forwardInfo
case timestamp
case locationLatitude
case locationLongitude
case expiresOn
}
public let path: String
public let isVideo: Bool
public let thumbnail: UIImage
public let dimensions: PixelDimensions
public let duration: Double?
public let values: MediaEditorValues
public let caption: NSAttributedString
public let privacy: MediaEditorResultPrivacy?
public let forwardInfo: StoryId?
public let timestamp: Int32
public let location: CLLocationCoordinate2D?
public let expiresOn: Int32?
public init(path: String, isVideo: Bool, thumbnail: UIImage, dimensions: PixelDimensions, duration: Double?, values: MediaEditorValues, caption: NSAttributedString, privacy: MediaEditorResultPrivacy?, forwardInfo: StoryId?, timestamp: Int32, location: CLLocationCoordinate2D?, expiresOn: Int32?) {
self.path = path
self.isVideo = isVideo
self.thumbnail = thumbnail
self.dimensions = dimensions
self.duration = duration
self.values = values
self.caption = caption
self.privacy = privacy
self.forwardInfo = forwardInfo
self.timestamp = timestamp
self.location = location
self.expiresOn = expiresOn
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.path = try container.decode(String.self, forKey: .path)
self.isVideo = try container.decode(Bool.self, forKey: .isVideo)
let thumbnailData = try container.decode(Data.self, forKey: .thumbnail)
if let thumbnail = UIImage(data: thumbnailData) {
self.thumbnail = thumbnail
} else {
throw ReadError.generic
}
self.dimensions = PixelDimensions(
width: try container.decode(Int32.self, forKey: .dimensionsWidth),
height: try container.decode(Int32.self, forKey: .dimensionsHeight)
)
self.duration = try container.decodeIfPresent(Double.self, forKey: .duration)
let valuesData = try container.decode(Data.self, forKey: .values)
if let values = try? JSONDecoder().decode(MediaEditorValues.self, from: valuesData) {
self.values = values
} else {
throw ReadError.generic
}
self.caption = ((try? container.decode(ChatTextInputStateText.self, forKey: .caption)) ?? ChatTextInputStateText()).attributedText()
if let data = try container.decodeIfPresent(Data.self, forKey: .privacy), let privacy = try? JSONDecoder().decode(MediaEditorResultPrivacy.self, from: data) {
self.privacy = privacy
} else {
self.privacy = nil
}
self.forwardInfo = try container.decodeIfPresent(StoryId.self, forKey: .forwardInfo)
self.timestamp = try container.decodeIfPresent(Int32.self, forKey: .timestamp) ?? 1688909663
if let latitude = try container.decodeIfPresent(Double.self, forKey: .locationLatitude), let longitude = try container.decodeIfPresent(Double.self, forKey: .locationLongitude) {
self.location = CLLocationCoordinate2D(latitude: latitude, longitude: longitude)
} else {
self.location = nil
}
self.expiresOn = try container.decodeIfPresent(Int32.self, forKey: .expiresOn)
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.path, forKey: .path)
try container.encode(self.isVideo, forKey: .isVideo)
if let thumbnailData = self.thumbnail.jpegData(compressionQuality: 0.6) {
try container.encode(thumbnailData, forKey: .thumbnail)
}
try container.encode(self.dimensions.width, forKey: .dimensionsWidth)
try container.encode(self.dimensions.height, forKey: .dimensionsHeight)
try container.encodeIfPresent(self.duration, forKey: .duration)
if let valuesData = try? JSONEncoder().encode(self.values) {
try container.encode(valuesData, forKey: .values)
}
let chatInputText = ChatTextInputStateText(attributedText: self.caption)
try container.encode(chatInputText, forKey: .caption)
if let privacy = self.privacy {
if let data = try? JSONEncoder().encode(privacy) {
try container.encode(data, forKey: .privacy)
} else {
try container.encodeNil(forKey: .privacy)
}
} else {
try container.encodeNil(forKey: .privacy)
}
try container.encodeIfPresent(self.forwardInfo, forKey: .forwardInfo)
try container.encode(self.timestamp, forKey: .timestamp)
if let location = self.location {
try container.encode(location.latitude, forKey: .locationLatitude)
try container.encode(location.longitude, forKey: .locationLongitude)
} else {
try container.encodeNil(forKey: .locationLatitude)
try container.encodeNil(forKey: .locationLongitude)
}
try container.encodeIfPresent(self.expiresOn, forKey: .expiresOn)
}
}
private struct MediaEditorDraftItemId {
public let rawValue: MemoryBuffer
var value: Int64 {
return self.rawValue.makeData().withUnsafeBytes { buffer -> Int64 in
guard let bytes = buffer.baseAddress?.assumingMemoryBound(to: Int64.self) else {
return 0
}
return bytes.pointee
}
}
init(_ rawValue: MemoryBuffer) {
self.rawValue = rawValue
}
init(_ value: Int64) {
var value = value
self.rawValue = MemoryBuffer(data: Data(bytes: &value, count: MemoryLayout.size(ofValue: value)))
}
init(_ value: UInt64) {
var value = Int64(bitPattern: value)
self.rawValue = MemoryBuffer(data: Data(bytes: &value, count: MemoryLayout.size(ofValue: value)))
}
}
public func addStoryDraft(engine: TelegramEngine, item: MediaEditorDraft) {
let itemId = MediaEditorDraftItemId(item.path.persistentHashValue)
let _ = engine.orderedLists.addOrMoveToFirstPosition(collectionId: ApplicationSpecificOrderedItemListCollectionId.storyDrafts, id: itemId.rawValue, item: item, removeTailIfCountExceeds: 50).start()
}
public func removeStoryDraft(engine: TelegramEngine, path: String, delete: Bool) {
if delete {
try? FileManager.default.removeItem(atPath: fullDraftPath(peerId: engine.account.peerId, path: path))
}
let itemId = MediaEditorDraftItemId(path.persistentHashValue)
let _ = engine.orderedLists.removeItem(collectionId: ApplicationSpecificOrderedItemListCollectionId.storyDrafts, id: itemId.rawValue).start()
}
public func clearStoryDrafts(engine: TelegramEngine) {
let _ = engine.data.get(TelegramEngine.EngineData.Item.OrderedLists.ListItems(collectionId: ApplicationSpecificOrderedItemListCollectionId.storyDrafts)).start(next: { items in
for item in items {
if let draft = item.contents.get(MediaEditorDraft.self) {
removeStoryDraft(engine: engine, path: draft.path, delete: true)
}
}
})
}
public func deleteAllStoryDrafts(peerId: EnginePeer.Id) {
let path = NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0] + "/storyDrafts_\(peerId.toInt64())/"
try? FileManager.default.removeItem(atPath: path)
}
public func storyDrafts(engine: TelegramEngine) -> Signal<[MediaEditorDraft], NoError> {
return engine.data.subscribe(TelegramEngine.EngineData.Item.OrderedLists.ListItems(collectionId: ApplicationSpecificOrderedItemListCollectionId.storyDrafts))
|> map { items -> [MediaEditorDraft] in
var result: [MediaEditorDraft] = []
for item in items {
if let draft = item.contents.get(MediaEditorDraft.self) {
result.append(draft)
}
}
return result
}
}
public func updateStoryDrafts(engine: TelegramEngine) {
let currentTimestamp = Int32(CFAbsoluteTimeGetCurrent() + kCFAbsoluteTimeIntervalSince1970)
let _ = engine.data.get(
TelegramEngine.EngineData.Item.OrderedLists.ListItems(collectionId: ApplicationSpecificOrderedItemListCollectionId.storyDrafts)
).start(next: { items in
for item in items {
if let draft = item.contents.get(MediaEditorDraft.self) {
if let expiresOn = draft.expiresOn, expiresOn < currentTimestamp {
removeStoryDraft(engine: engine, path: draft.path, delete: true)
}
}
}
})
}
public extension MediaEditorDraft {
func fullPath(engine: TelegramEngine) -> String {
return fullDraftPath(peerId: engine.account.peerId, path: self.path)
}
}
func fullDraftPath(peerId: EnginePeer.Id, path: String) -> String {
return NSSearchPathForDirectoriesInDomains(.documentDirectory, .userDomainMask, true)[0] + "/storyDrafts_\(peerId.toInt64())/" + path
}
@@ -0,0 +1,81 @@
import Foundation
import UIKit
import Metal
import MetalKit
import SwiftSignalKit
public final class MediaEditorPreviewView: MTKView, MTKViewDelegate, RenderTarget {
weak var renderer: MediaEditorRenderer? {
didSet {
if let renderer = self.renderer {
renderer.renderTargetDidChange(self)
}
}
}
var drawable: MTLDrawable? {
return self.nextDrawable
}
var nextDrawable: MTLDrawable? {
if #available(iOS 13.0, *) {
if let layer = self.layer as? CAMetalLayer {
return layer.nextDrawable()
} else {
return self.currentDrawable
}
} else {
return self.currentDrawable
}
}
var renderPassDescriptor: MTLRenderPassDescriptor? {
return self.currentRenderPassDescriptor
}
var mtlDevice: MTLDevice? {
return self.device
}
public override init(frame frameRect: CGRect, device: MTLDevice?) {
super.init(frame: frameRect, device: device)
self.setup()
}
required init(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
private func setup() {
guard let device = MTLCreateSystemDefaultDevice() else {
return
}
self.device = device
self.delegate = self
self.colorPixelFormat = .bgra8Unorm
self.isPaused = true
self.enableSetNeedsDisplay = true
self.framebufferOnly = true
}
public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
Queue.mainQueue().justDispatch {
self.renderer?.renderTargetDrawableSizeDidChange(size)
}
}
public func redraw() {
self.setNeedsDisplay()
}
public func draw(in view: MTKView) {
guard self.frame.width > 0.0 else {
return
}
self.renderer?.displayFrame()
}
}
@@ -0,0 +1,153 @@
import Foundation
import simd
final class MediaEditorRenderChain {
let enhancePass = EnhanceRenderPass()
let sharpenPass = SharpenRenderPass()
let blurPass = BlurRenderPass()
let adjustmentsPass = AdjustmentsRenderPass()
let stickerOutlinePass = StickerOutlineRenderPass()
var renderPasses: [RenderPass] {
return [
self.enhancePass,
self.sharpenPass,
self.blurPass,
self.adjustmentsPass,
self.stickerOutlinePass
]
}
func update(values: MediaEditorValues) {
for key in EditorToolKey.allCases {
let value = values.toolValues[key]
switch key {
case .enhance:
if let value = value as? Float {
self.enhancePass.value = abs(value)
} else {
self.enhancePass.value = 0.0
}
case .brightness:
if let value = value as? Float {
self.adjustmentsPass.adjustments.exposure = value
} else {
self.adjustmentsPass.adjustments.exposure = 0.0
}
case .contrast:
if let value = value as? Float {
self.adjustmentsPass.adjustments.contrast = value
} else {
self.adjustmentsPass.adjustments.contrast = 0.0
}
case .saturation:
if let value = value as? Float {
self.adjustmentsPass.adjustments.saturation = value
} else {
self.adjustmentsPass.adjustments.saturation = 0.0
}
case .warmth:
if let value = value as? Float {
self.adjustmentsPass.adjustments.warmth = value
} else {
self.adjustmentsPass.adjustments.warmth = 0.0
}
case .fade:
if let value = value as? Float {
self.adjustmentsPass.adjustments.fade = value
} else {
self.adjustmentsPass.adjustments.fade = 0.0
}
case .highlights:
if let value = value as? Float {
self.adjustmentsPass.adjustments.highlights = value
} else {
self.adjustmentsPass.adjustments.highlights = 0.0
}
case .shadows:
if let value = value as? Float {
self.adjustmentsPass.adjustments.shadows = value
} else {
self.adjustmentsPass.adjustments.shadows = 0.0
}
case .vignette:
if let value = value as? Float {
self.adjustmentsPass.adjustments.vignette = value
} else {
self.adjustmentsPass.adjustments.vignette = 0.0
}
case .grain:
if let value = value as? Float {
self.adjustmentsPass.adjustments.grain = value
} else {
self.adjustmentsPass.adjustments.grain = 0.0
}
case .sharpen:
if let value = value as? Float {
self.sharpenPass.value = value
} else {
self.sharpenPass.value = 0.0
}
case .shadowsTint:
if let value = value as? TintValue {
if value.color != .clear {
let (red, green, blue, _) = value.color.components
self.adjustmentsPass.adjustments.shadowsTintColor = simd_float3(Float(red), Float(green), Float(blue))
self.adjustmentsPass.adjustments.shadowsTintIntensity = value.intensity
} else {
self.adjustmentsPass.adjustments.shadowsTintIntensity = 0.0
}
}
case .highlightsTint:
if let value = value as? TintValue {
if value.color != .clear {
let (red, green, blue, _) = value.color.components
self.adjustmentsPass.adjustments.shadowsTintColor = simd_float3(Float(red), Float(green), Float(blue))
self.adjustmentsPass.adjustments.highlightsTintIntensity = value.intensity
} else {
self.adjustmentsPass.adjustments.highlightsTintIntensity = 0.0
}
}
case .blur:
if let value = value as? BlurValue {
switch value.mode {
case .off:
self.blurPass.mode = .off
case .linear:
self.blurPass.mode = .linear
case .radial:
self.blurPass.mode = .radial
case .portrait:
self.blurPass.mode = .portrait
}
self.blurPass.intensity = value.intensity
self.blurPass.value.size = Float(value.size)
self.blurPass.value.position = simd_float2(Float(value.position.x), Float(value.position.y))
self.blurPass.value.falloff = Float(value.falloff)
self.blurPass.value.rotation = Float(value.rotation)
}
case .curves:
if var value = value as? CurvesValue {
let allDataPoints = value.all.dataPoints
let redDataPoints = value.red.dataPoints
let greenDataPoints = value.green.dataPoints
let blueDataPoints = value.blue.dataPoints
self.adjustmentsPass.adjustments.hasCurves = 1.0
self.adjustmentsPass.allCurve = allDataPoints
self.adjustmentsPass.redCurve = redDataPoints
self.adjustmentsPass.greenCurve = greenDataPoints
self.adjustmentsPass.blueCurve = blueDataPoints
} else {
self.adjustmentsPass.adjustments.hasCurves = 0.0
}
case .stickerOutline:
if let value = value as? Float {
self.stickerOutlinePass.value = value
} else {
self.stickerOutlinePass.value = 0.0
}
}
}
}
}
@@ -0,0 +1,367 @@
import Foundation
import UIKit
import Metal
import MetalKit
import Photos
import SwiftSignalKit
public final class VideoPixelBuffer {
let pixelBuffer: CVPixelBuffer
let rotation: TextureRotation
let timestamp: CMTime
public init(
pixelBuffer: CVPixelBuffer,
rotation: TextureRotation,
timestamp: CMTime
) {
self.pixelBuffer = pixelBuffer
self.rotation = rotation
self.timestamp = timestamp
}
}
protocol RenderPass: AnyObject {
func setup(device: MTLDevice, library: MTLLibrary)
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture?
}
protocol TextureSource {
func connect(to renderer: MediaEditorRenderer)
func invalidate()
func setRate(_ rate: Float)
}
protocol RenderTarget: AnyObject {
var mtlDevice: MTLDevice? { get }
var drawableSize: CGSize { get }
var colorPixelFormat: MTLPixelFormat { get }
var drawable: MTLDrawable? { get }
var renderPassDescriptor: MTLRenderPassDescriptor? { get }
func redraw()
}
final class MediaEditorRenderer {
enum Input {
case texture(MTLTexture, CMTime, Bool, CGRect?, CGFloat, CGPoint)
case videoBuffer(VideoPixelBuffer, CGRect?, CGFloat, CGPoint)
case ciImage(CIImage, CMTime)
var timestamp: CMTime {
switch self {
case let .texture(_, timestamp, _, _, _, _):
return timestamp
case let .videoBuffer(videoBuffer, _, _, _):
return videoBuffer.timestamp
case let .ciImage(_, timestamp):
return timestamp
}
}
}
private var device: MTLDevice?
private var library: MTLLibrary?
private var commandQueue: MTLCommandQueue?
private var textureCache: CVMetalTextureCache?
private var semaphore = DispatchSemaphore(value: 3)
private var renderPasses: [RenderPass] = []
private let ciInputPass = CIInputPass()
private let mainVideoInputPass = VideoInputPass()
private var additionalVideoInputPass: [Int : VideoInputPass] = [:]
let videoFinishPass = VideoFinishPass()
private let outputRenderPass = OutputRenderPass()
private weak var renderTarget: RenderTarget? {
didSet {
self.outputRenderPass.renderTarget = self.renderTarget
}
}
var textureSource: TextureSource? {
didSet {
self.textureSource?.connect(to: self)
}
}
private var currentMainInput: Input?
var currentMainInputMask: MTLTexture?
private var currentAdditionalInputs: [Input] = []
private(set) var resultTexture: MTLTexture?
var displayEnabled = true
var skipEditingPasses = false
var needsDisplay = false
var onNextRender: (() -> Void)?
var onNextAdditionalRender: (() -> Void)?
public init() {
}
deinit {
for _ in 0 ..< 3 {
self.semaphore.signal()
}
}
func addRenderPass(_ renderPass: RenderPass) {
self.renderPasses.append(renderPass)
if let device = self.effectiveDevice, let library = self.library {
renderPass.setup(device: device, library: library)
}
}
func addRenderChain(_ renderChain: MediaEditorRenderChain) {
for renderPass in renderChain.renderPasses {
self.addRenderPass(renderPass)
}
}
private func commonSetup(device: MTLDevice) {
CVMetalTextureCacheCreate(nil, nil, device, nil, &self.textureCache)
let mainBundle = Bundle(for: MediaEditorRenderer.self)
guard let path = mainBundle.path(forResource: "MediaEditorBundle", ofType: "bundle") else {
return
}
guard let bundle = Bundle(path: path) else {
return
}
guard let library = try? device.makeDefaultLibrary(bundle: bundle) else {
return
}
self.library = library
self.commandQueue = device.makeCommandQueue()
self.commandQueue?.label = "Media Editor Command Queue"
self.ciInputPass.setup(device: device, library: library)
self.mainVideoInputPass.setup(device: device, library: library)
self.videoFinishPass.setup(device: device, library: library)
self.renderPasses.forEach { $0.setup(device: device, library: library) }
}
var effectiveDevice: MTLDevice? {
if let device = self.renderTarget?.mtlDevice {
return device
} else {
return self.device
}
}
private func setup() {
guard let device = self.renderTarget?.mtlDevice else {
return
}
self.commonSetup(device: device)
guard let library = self.library else {
return
}
self.outputRenderPass.setup(device: device, library: library)
}
func setupForComposer(composer: MediaEditorComposer) {
guard let device = composer.device else {
return
}
self.device = device
self.commonSetup(device: device)
}
func setupForStandaloneDevice(device: MTLDevice) {
self.device = device
self.commonSetup(device: device)
}
func setRate(_ rate: Float) {
self.textureSource?.setRate(rate)
}
private func combinedTextureFromCurrentInputs(device: MTLDevice, commandBuffer: MTLCommandBuffer, textureCache: CVMetalTextureCache) -> MTLTexture? {
guard let library = self.library else {
return nil
}
var passMainInput: VideoFinishPass.Input?
var passAdditionalInputs: [VideoFinishPass.Input] = []
func textureFromInput(_ input: MediaEditorRenderer.Input, videoInputPass: VideoInputPass) -> VideoFinishPass.Input? {
switch input {
case let .texture(texture, _, hasTransparency, rect, scale, offset):
return VideoFinishPass.Input(texture: texture, hasTransparency: hasTransparency, rect: rect, scale: scale, offset: offset)
case let .videoBuffer(videoBuffer, rect, scale, offset):
if let texture = videoInputPass.processPixelBuffer(videoBuffer, textureCache: textureCache, device: device, commandBuffer: commandBuffer) {
return VideoFinishPass.Input(texture: texture, hasTransparency: false, rect: rect, scale: scale, offset: offset)
} else {
return nil
}
case let .ciImage(image, _):
if let texture = self.ciInputPass.processCIImage(image, device: device, commandBuffer: commandBuffer) {
return VideoFinishPass.Input(texture: texture, hasTransparency: true, rect: nil, scale: 1.0, offset: .zero)
} else {
return nil
}
}
}
guard let mainInput = self.currentMainInput else {
return nil
}
if let input = textureFromInput(mainInput, videoInputPass: self.mainVideoInputPass) {
passMainInput = input
}
var index = 0
for additionalInput in self.currentAdditionalInputs {
let videoInputPass: VideoInputPass
if let current = self.additionalVideoInputPass[index] {
videoInputPass = current
} else {
videoInputPass = VideoInputPass()
videoInputPass.setup(device: device, library: library)
self.additionalVideoInputPass[index] = videoInputPass
}
if let input = textureFromInput(additionalInput, videoInputPass: videoInputPass) {
passAdditionalInputs.append(input)
}
index += 1
}
if let passMainInput {
return self.videoFinishPass.process(input: passMainInput, inputMask: self.currentMainInputMask, hasTransparency: passMainInput.hasTransparency, secondInput: passAdditionalInputs, timestamp: mainInput.timestamp, device: device, commandBuffer: commandBuffer)
} else {
return nil
}
}
func renderFrame() {
guard let device = self.effectiveDevice,
let commandQueue = self.commandQueue,
let textureCache = self.textureCache,
let commandBuffer = commandQueue.makeCommandBuffer(),
var texture = self.combinedTextureFromCurrentInputs(device: device, commandBuffer: commandBuffer, textureCache: textureCache)
else {
self.didRenderFrame()
return
}
if !self.skipEditingPasses {
for renderPass in self.renderPasses {
if let nextTexture = renderPass.process(input: texture, device: device, commandBuffer: commandBuffer) {
texture = nextTexture
}
}
}
self.resultTexture = texture
if self.renderTarget == nil {
commandBuffer.addCompletedHandler { [weak self] _ in
if let self {
self.didRenderFrame()
}
}
}
commandBuffer.commit()
if let renderTarget = self.renderTarget, self.displayEnabled {
if self.needsDisplay {
self.didRenderFrame()
} else {
self.needsDisplay = true
renderTarget.redraw()
}
} else {
commandBuffer.waitUntilCompleted()
}
}
func displayFrame() {
guard let renderTarget = self.renderTarget,
let device = renderTarget.mtlDevice,
let commandQueue = self.commandQueue,
let commandBuffer = commandQueue.makeCommandBuffer(),
let texture = self.resultTexture
else {
self.needsDisplay = false
self.didRenderFrame()
return
}
commandBuffer.addCompletedHandler { [weak self] _ in
if let self {
self.didRenderFrame()
if let onNextRender = self.onNextRender {
self.onNextRender = nil
Queue.mainQueue().after(0.016) {
onNextRender()
}
}
if let onNextAdditionalRender = self.onNextAdditionalRender {
if !self.currentAdditionalInputs.isEmpty {
self.onNextAdditionalRender = nil
Queue.mainQueue().after(0.016) {
onNextAdditionalRender()
}
}
}
}
}
self.outputRenderPass.process(input: texture, device: device, commandBuffer: commandBuffer)
commandBuffer.commit()
self.needsDisplay = false
}
func willRenderFrame() {
let timeout = self.renderTarget != nil ? DispatchTime.now() + 0.1 : .distantFuture
let _ = self.semaphore.wait(timeout: timeout)
}
func didRenderFrame() {
self.semaphore.signal()
}
func consume(
main: MediaEditorRenderer.Input,
additionals: [MediaEditorRenderer.Input],
render: Bool,
displayEnabled: Bool = true
) {
self.displayEnabled = displayEnabled
if render {
self.willRenderFrame()
}
self.currentMainInput = main
self.currentAdditionalInputs = additionals
if render {
self.renderFrame()
}
}
func renderTargetDidChange(_ target: RenderTarget?) {
self.renderTarget = target
self.setup()
}
func renderTargetDrawableSizeDidChange(_ size: CGSize) {
self.renderTarget?.redraw()
}
func finalRenderedImage(mirror: Bool = false) -> UIImage? {
if let finalTexture = self.resultTexture, let device = self.effectiveDevice {
return getTextureImage(device: device, texture: finalTexture, mirror: mirror)
} else {
return nil
}
}
}
@@ -0,0 +1,138 @@
import Foundation
import UIKit
import SwiftSignalKit
import TelegramCore
import TelegramUIPreferences
public final class MediaEditorStoredTextSettings: Codable {
private enum CodingKeys: String, CodingKey {
case style
case font
case fontSize
case alignment
}
public let style: DrawingTextEntity.Style
public let font: DrawingTextEntity.Font
public let fontSize: CGFloat
public let alignment: DrawingTextEntity.Alignment
public init(
style: DrawingTextEntity.Style,
font: DrawingTextEntity.Font,
fontSize: CGFloat,
alignment: DrawingTextEntity.Alignment
) {
self.style = style
self.font = font
self.fontSize = fontSize
self.alignment = alignment
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
self.style = try container.decode(DrawingTextEntity.Style.self, forKey: .style)
self.font = try container.decode(DrawingTextEntity.Font.self, forKey: .font)
self.fontSize = try container.decode(CGFloat.self, forKey: .fontSize)
self.alignment = try container.decode(DrawingTextEntity.Alignment.self, forKey: .alignment)
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(self.style, forKey: .style)
try container.encode(self.font, forKey: .font)
try container.encode(self.fontSize, forKey: .fontSize)
try container.encode(self.alignment, forKey: .alignment)
}
}
public final class MediaEditorStoredState: Codable {
private enum CodingKeys: String, CodingKey {
case privacy
case textSettings
}
public let privacy: MediaEditorResultPrivacy?
public let textSettings: MediaEditorStoredTextSettings?
public init(privacy: MediaEditorResultPrivacy?, textSettings: MediaEditorStoredTextSettings?) {
self.privacy = privacy
self.textSettings = textSettings
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
if let data = try container.decodeIfPresent(Data.self, forKey: .privacy), let privacy = try? JSONDecoder().decode(MediaEditorResultPrivacy.self, from: data) {
self.privacy = privacy
} else {
self.privacy = nil
}
if let data = try container.decodeIfPresent(Data.self, forKey: .textSettings), let privacy = try? JSONDecoder().decode(MediaEditorStoredTextSettings.self, from: data) {
self.textSettings = privacy
} else {
self.textSettings = nil
}
}
public func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
if let privacy = self.privacy {
if let data = try? JSONEncoder().encode(privacy) {
try container.encode(data, forKey: .privacy)
} else {
try container.encodeNil(forKey: .privacy)
}
} else {
try container.encodeNil(forKey: .privacy)
}
if let textSettings = self.textSettings {
if let data = try? JSONEncoder().encode(textSettings) {
try container.encode(data, forKey: .textSettings)
} else {
try container.encodeNil(forKey: .textSettings)
}
} else {
try container.encodeNil(forKey: .textSettings)
}
}
public func withUpdatedPrivacy(_ privacy: MediaEditorResultPrivacy) -> MediaEditorStoredState {
return MediaEditorStoredState(privacy: privacy, textSettings: self.textSettings)
}
public func withUpdatedTextSettings(_ textSettings: MediaEditorStoredTextSettings) -> MediaEditorStoredState {
return MediaEditorStoredState(privacy: self.privacy, textSettings: textSettings)
}
}
public func mediaEditorStoredState(engine: TelegramEngine) -> Signal<MediaEditorStoredState?, NoError> {
let key = EngineDataBuffer(length: 4)
key.setInt32(0, value: 0)
return engine.data.get(TelegramEngine.EngineData.Item.ItemCache.Item(collectionId: ApplicationSpecificItemCacheCollectionId.mediaEditorState, id: key))
|> map { entry -> MediaEditorStoredState? in
return entry?.get(MediaEditorStoredState.self)
}
}
public func updateMediaEditorStoredStateInteractively(engine: TelegramEngine, _ f: @escaping (MediaEditorStoredState?) -> MediaEditorStoredState?) -> Signal<Never, NoError> {
let key = EngineDataBuffer(length: 4)
key.setInt32(0, value: 0)
return engine.data.get(TelegramEngine.EngineData.Item.ItemCache.Item(collectionId: ApplicationSpecificItemCacheCollectionId.mediaEditorState, id: key))
|> map { entry -> MediaEditorStoredState? in
return entry?.get(MediaEditorStoredState.self)
}
|> mapToSignal { state -> Signal<Never, NoError> in
if let updatedState = f(state) {
return engine.itemCache.put(collectionId: ApplicationSpecificItemCacheCollectionId.mediaEditorState, id: key, item: updatedState)
} else {
return engine.itemCache.remove(collectionId: ApplicationSpecificItemCacheCollectionId.mediaEditorState, id: key)
}
}
}
@@ -0,0 +1,245 @@
import Foundation
import UIKit
import AVFoundation
import SwiftSignalKit
import TelegramCore
import AccountContext
import TelegramUIPreferences
import TelegramPresentationData
extension AVPlayer {
func fadeVolume(from: Float, to: Float, duration: Float, completion: (() -> Void)? = nil) -> SwiftSignalKit.Timer? {
self.volume = from
guard from != to else { return nil }
let interval: Float = 0.1
let range = to - from
let step = (range * interval) / duration
func reachedTarget() -> Bool {
guard self.volume >= 0, self.volume <= 1 else {
self.volume = to
return true
}
if to > from {
return self.volume >= to
}
return self.volume <= to
}
var invalidateImpl: (() -> Void)?
let timer = SwiftSignalKit.Timer(timeout: Double(interval), repeat: true, completion: { [weak self] in
if let self, !reachedTarget() {
self.volume += step
} else {
invalidateImpl?()
completion?()
}
}, queue: Queue.mainQueue())
invalidateImpl = { [weak timer] in
timer?.invalidate()
}
timer.start()
return timer
}
}
func textureRotatonForAVAsset(_ asset: AVAsset, mirror: Bool = false) -> TextureRotation {
for track in asset.tracks {
if track.mediaType == .video {
let t = track.preferredTransform
if t.a == -1.0 && t.d == -1.0 {
return .rotate180Degrees
} else if t.a == 1.0 && t.d == 1.0 {
return .rotate0Degrees
} else if t.b == -1.0 && t.c == 1.0 {
return .rotate270Degrees
} else if t.a == -1.0 && t.d == 1.0 {
return .rotate270Degrees
} else if t.a == 1.0 && t.d == -1.0 {
return .rotate180Degrees
} else {
return mirror ? .rotate90DegreesMirrored : .rotate90Degrees
}
}
}
return .rotate0Degrees
}
func loadTexture(image: UIImage, device: MTLDevice) -> MTLTexture? {
func dataForImage(_ image: UIImage) -> UnsafeMutablePointer<UInt8> {
let imageRef = image.cgImage
let width = Int(image.size.width)
let height = Int(image.size.height)
let colorSpace = CGColorSpaceCreateDeviceRGB()
let rawData = UnsafeMutablePointer<UInt8>.allocate(capacity: width * height * 4)
let bytePerPixel = 4
let bytesPerRow = bytePerPixel * Int(width)
let bitsPerComponent = 8
let bitmapInfo = CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue)
if let context = CGContext(data: rawData, width: width, height: height, bitsPerComponent: bitsPerComponent, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo.rawValue) {
context.clear(CGRect(x: 0, y: 0, width: width, height: height))
context.draw(imageRef!, in: CGRect(x: 0, y: 0, width: width, height: height))
}
return rawData
}
let width = Int(image.size.width * image.scale)
let height = Int(image.size.height * image.scale)
let bytePerPixel = 4
let bytesPerRow = bytePerPixel * width
var texture : MTLTexture?
let region = MTLRegionMake2D(0, 0, Int(width), Int(height))
let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm, width: width, height: height, mipmapped: false)
texture = device.makeTexture(descriptor: textureDescriptor)
let data = dataForImage(image)
texture?.replace(region: region, mipmapLevel: 0, withBytes: data, bytesPerRow: bytesPerRow)
data.deallocate()
return texture
}
func pixelBufferToMTLTexture(pixelBuffer: CVPixelBuffer, textureCache: CVMetalTextureCache) -> MTLTexture? {
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let format: MTLPixelFormat = .r8Unorm
var textureRef : CVMetalTexture?
let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, format, width, height, 0, &textureRef)
if status == kCVReturnSuccess {
return CVMetalTextureGetTexture(textureRef!)
}
return nil
}
func getTextureImage(device: MTLDevice, texture: MTLTexture, mirror: Bool = false) -> UIImage? {
let colorSpace = CGColorSpaceCreateDeviceRGB()
let context = CIContext(mtlDevice: device, options: [:])
guard var ciImage = CIImage(mtlTexture: texture, options: [.colorSpace: colorSpace]) else {
return nil
}
let transform: CGAffineTransform
if mirror {
transform = CGAffineTransform(-1.0, 0.0, 0.0, -1.0, ciImage.extent.width, ciImage.extent.height)
} else {
transform = CGAffineTransform(1.0, 0.0, 0.0, -1.0, 0.0, ciImage.extent.height)
}
ciImage = ciImage.transformed(by: transform)
guard let cgImage = context.createCGImage(ciImage, from: CGRect(origin: .zero, size: CGSize(width: ciImage.extent.width, height: ciImage.extent.height))) else {
return nil
}
return UIImage(cgImage: cgImage)
}
public func getChatWallpaperImage(context: AccountContext, peerId: EnginePeer.Id) -> Signal<(CGSize, UIImage?, UIImage?), NoError> {
let themeSettings = context.sharedContext.accountManager.sharedData(keys: [ApplicationSpecificSharedDataKeys.presentationThemeSettings])
|> map { sharedData -> PresentationThemeSettings in
let themeSettings: PresentationThemeSettings
if let current = sharedData.entries[ApplicationSpecificSharedDataKeys.presentationThemeSettings]?.get(PresentationThemeSettings.self) {
themeSettings = current
} else {
themeSettings = PresentationThemeSettings.defaultSettings
}
return themeSettings
}
let peerWallpaper = context.account.postbox.transaction { transaction -> TelegramWallpaper? in
return (transaction.getPeerCachedData(peerId: peerId) as? CachedChannelData)?.wallpaper
}
return combineLatest(themeSettings, peerWallpaper)
|> mapToSignal { themeSettings, peerWallpaper -> Signal<(TelegramWallpaper?, TelegramWallpaper?), NoError> in
var currentColors = themeSettings.themeSpecificAccentColors[themeSettings.theme.index]
if let colors = currentColors, colors.baseColor == .theme {
currentColors = nil
}
let themeSpecificWallpaper = (themeSettings.themeSpecificChatWallpapers[coloredThemeIndex(reference: themeSettings.theme, accentColor: currentColors)] ?? themeSettings.themeSpecificChatWallpapers[themeSettings.theme.index])
let dayWallpaper: TelegramWallpaper
if let themeSpecificWallpaper = themeSpecificWallpaper {
dayWallpaper = themeSpecificWallpaper
} else {
let theme = makePresentationTheme(mediaBox: context.sharedContext.accountManager.mediaBox, themeReference: themeSettings.theme, accentColor: currentColors?.color, bubbleColors: currentColors?.customBubbleColors ?? [], wallpaper: currentColors?.wallpaper, baseColor: currentColors?.baseColor, preview: true) ?? defaultPresentationTheme
dayWallpaper = theme.chat.defaultWallpaper
}
var nightWallpaper: TelegramWallpaper?
let automaticTheme = themeSettings.automaticThemeSwitchSetting.theme
let effectiveColors = themeSettings.themeSpecificAccentColors[automaticTheme.index]
let nightThemeSpecificWallpaper = (themeSettings.themeSpecificChatWallpapers[coloredThemeIndex(reference: automaticTheme, accentColor: effectiveColors)] ?? themeSettings.themeSpecificChatWallpapers[automaticTheme.index])
var preferredBaseTheme: TelegramBaseTheme?
if let baseTheme = themeSettings.themePreferredBaseTheme[automaticTheme.index], [.night, .tinted].contains(baseTheme) {
preferredBaseTheme = baseTheme
} else {
preferredBaseTheme = .night
}
let darkTheme = makePresentationTheme(mediaBox: context.sharedContext.accountManager.mediaBox, themeReference: automaticTheme, baseTheme: preferredBaseTheme, accentColor: effectiveColors?.color, bubbleColors: effectiveColors?.customBubbleColors ?? [], wallpaper: effectiveColors?.wallpaper, baseColor: effectiveColors?.baseColor, serviceBackgroundColor: defaultServiceBackgroundColor) ?? defaultPresentationTheme
if let nightThemeSpecificWallpaper = nightThemeSpecificWallpaper {
nightWallpaper = nightThemeSpecificWallpaper
} else {
switch dayWallpaper {
case .builtin, .color, .gradient:
nightWallpaper = darkTheme.chat.defaultWallpaper
case .file:
if dayWallpaper.isPattern {
nightWallpaper = darkTheme.chat.defaultWallpaper
} else {
nightWallpaper = nil
}
default:
nightWallpaper = nil
}
}
if let peerWallpaper {
if case let .emoticon(emoticon) = peerWallpaper {
return context.engine.themes.getChatThemes(accountManager: context.sharedContext.accountManager)
|> map { themes -> (TelegramWallpaper?, TelegramWallpaper?) in
if let theme = themes.first(where: { $0.emoticon?.strippedEmoji == emoticon.strippedEmoji }) {
if let dayMatch = theme.settings?.first(where: { $0.baseTheme == .classic || $0.baseTheme == .day }) {
if let peerDayWallpaper = dayMatch.wallpaper {
var peerNightWallpaper: TelegramWallpaper?
if let nightMatch = theme.settings?.first(where: { $0.baseTheme == .night || $0.baseTheme == .tinted }) {
peerNightWallpaper = nightMatch.wallpaper
}
return (peerDayWallpaper, peerNightWallpaper)
} else {
return (dayWallpaper, nightWallpaper)
}
} else {
return (dayWallpaper, nightWallpaper)
}
} else {
return (dayWallpaper, nightWallpaper)
}
}
} else {
return .single((peerWallpaper, nil))
}
} else {
return .single((dayWallpaper, nightWallpaper))
}
}
|> mapToSignal { dayWallpaper, nightWallpaper -> Signal<(CGSize, UIImage?, UIImage?), NoError> in
return Signal { subscriber in
Queue.mainQueue().async {
let wallpaperRenderer = DrawingWallpaperRenderer(context: context, dayWallpaper: dayWallpaper, nightWallpaper: nightWallpaper)
wallpaperRenderer.render { size, image, darkImage, mediaRect in
subscriber.putNext((size, image, darkImage))
subscriber.putCompletion()
}
}
return EmptyDisposable
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,160 @@
import Foundation
import AVFoundation
import TelegramCore
import FFMpegBinding
final class MediaEditorVideoAVAssetWriter: MediaEditorVideoExportWriter {
private var writer: AVAssetWriter?
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private var adaptor: AVAssetWriterInputPixelBufferAdaptor!
func setup(configuration: MediaEditorVideoExport.Configuration, outputPath: String) {
Logger.shared.log("VideoExport", "Will setup asset writer")
let url = URL(fileURLWithPath: outputPath)
self.writer = try? AVAssetWriter(url: url, fileType: .mp4)
guard let writer = self.writer else {
return
}
writer.shouldOptimizeForNetworkUse = configuration.shouldOptimizeForNetworkUse
Logger.shared.log("VideoExport", "Did setup asset writer")
}
func setupVideoInput(configuration: MediaEditorVideoExport.Configuration, preferredTransform: CGAffineTransform?, sourceFrameRate: Float) {
guard let writer = self.writer else {
return
}
Logger.shared.log("VideoExport", "Will setup video input")
var dimensions = configuration.dimensions
var videoSettings = configuration.videoSettings
if var compressionSettings = videoSettings[AVVideoCompressionPropertiesKey] as? [String: Any] {
compressionSettings[AVVideoExpectedSourceFrameRateKey] = sourceFrameRate
videoSettings[AVVideoCompressionPropertiesKey] = compressionSettings
}
if let preferredTransform {
if (preferredTransform.b == -1 && preferredTransform.c == 1) || (preferredTransform.b == 1 && preferredTransform.c == -1) {
dimensions = CGSize(width: dimensions.height, height: dimensions.width)
}
videoSettings[AVVideoWidthKey] = Int(dimensions.width)
videoSettings[AVVideoHeightKey] = Int(dimensions.height)
}
let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
if let preferredTransform {
videoInput.transform = preferredTransform
}
videoInput.expectsMediaDataInRealTime = false
let sourcePixelBufferAttributes = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
kCVPixelBufferWidthKey as String: UInt32(dimensions.width),
kCVPixelBufferHeightKey as String: UInt32(dimensions.height)
]
self.adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoInput, sourcePixelBufferAttributes: sourcePixelBufferAttributes)
if writer.canAdd(videoInput) {
writer.add(videoInput)
} else {
Logger.shared.log("VideoExport", "Failed to add video input")
}
self.videoInput = videoInput
}
func setupAudioInput(configuration: MediaEditorVideoExport.Configuration) {
guard let writer = self.writer else {
return
}
let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: configuration.audioSettings)
audioInput.expectsMediaDataInRealTime = false
if writer.canAdd(audioInput) {
writer.add(audioInput)
}
self.audioInput = audioInput
}
func startWriting() -> Bool {
return self.writer?.startWriting() ?? false
}
func startSession(atSourceTime time: CMTime) {
self.writer?.startSession(atSourceTime: time)
}
func finishWriting(completion: @escaping () -> Void) {
self.writer?.finishWriting(completionHandler: completion)
}
func cancelWriting() {
self.writer?.cancelWriting()
}
func requestVideoDataWhenReady(on queue: DispatchQueue, using block: @escaping () -> Void) {
self.videoInput?.requestMediaDataWhenReady(on: queue, using: block)
}
func requestAudioDataWhenReady(on queue: DispatchQueue, using block: @escaping () -> Void) {
self.audioInput?.requestMediaDataWhenReady(on: queue, using: block)
}
var isReadyForMoreVideoData: Bool {
return self.videoInput?.isReadyForMoreMediaData ?? false
}
func appendVideoBuffer(_ buffer: CMSampleBuffer) -> Bool {
return self.videoInput?.append(buffer) ?? false
}
func appendPixelBuffer(_ pixelBuffer: CVPixelBuffer, at time: CMTime) -> Bool {
return self.adaptor.append(pixelBuffer, withPresentationTime: time)
}
var pixelBufferPool: CVPixelBufferPool? {
return self.adaptor.pixelBufferPool
}
func markVideoAsFinished() {
self.videoInput?.markAsFinished()
}
var isReadyForMoreAudioData: Bool {
return self.audioInput?.isReadyForMoreMediaData ?? false
}
func appendAudioBuffer(_ buffer: CMSampleBuffer) -> Bool {
return self.audioInput?.append(buffer) ?? false
}
func markAudioAsFinished() {
self.audioInput?.markAsFinished()
}
var status: ExportWriterStatus {
if let writer = self.writer {
switch writer.status {
case .unknown:
return .unknown
case .writing:
return .writing
case .completed:
return .completed
case .failed:
return .failed
case .cancelled:
return .cancelled
@unknown default:
fatalError()
}
} else {
return .unknown
}
}
var error: Error? {
return self.writer?.error
}
}
@@ -0,0 +1,152 @@
import Foundation
import UIKit
import CoreMedia
import FFMpegBinding
import ImageDCT
import Accelerate
final class MediaEditorVideoFFMpegWriter: MediaEditorVideoExportWriter {
public static let registerFFMpegGlobals: Void = {
FFMpegGlobals.initializeGlobals()
return
}()
let ffmpegWriter = FFMpegVideoWriter()
var pool: CVPixelBufferPool?
let conversionInfo: vImage_ARGBToYpCbCr
init() {
var pixelRange = vImage_YpCbCrPixelRange( Yp_bias: 16, CbCr_bias: 128, YpRangeMax: 235, CbCrRangeMax: 240, YpMax: 235, YpMin: 16, CbCrMax: 240, CbCrMin: 16)
var conversionInfo = vImage_ARGBToYpCbCr()
let _ = vImageConvert_ARGBToYpCbCr_GenerateConversion(kvImage_ARGBToYpCbCrMatrix_ITU_R_709_2, &pixelRange, &conversionInfo, kvImageARGB8888, kvImage420Yp8_Cb8_Cr8, vImage_Flags(kvImageNoFlags))
self.conversionInfo = conversionInfo
}
func setup(configuration: MediaEditorVideoExport.Configuration, outputPath: String) {
let _ = MediaEditorVideoFFMpegWriter.registerFFMpegGlobals
let width = Int32(configuration.dimensions.width)
let height = Int32(configuration.dimensions.height)
let bufferOptions: [String: Any] = [
kCVPixelBufferPoolMinimumBufferCountKey as String: 3 as NSNumber
]
let pixelBufferOptions: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA as NSNumber,
kCVPixelBufferWidthKey as String: UInt32(width),
kCVPixelBufferHeightKey as String: UInt32(height)
]
var pool: CVPixelBufferPool?
CVPixelBufferPoolCreate(nil, bufferOptions as CFDictionary, pixelBufferOptions as CFDictionary, &pool)
guard let pool else {
self.status = .failed
return
}
self.pool = pool
if !self.ffmpegWriter.setup(withOutputPath: outputPath, width: width, height: height, bitrate: 240 * 1000, framerate: 30) {
self.status = .failed
}
}
func setupVideoInput(configuration: MediaEditorVideoExport.Configuration, preferredTransform: CGAffineTransform?, sourceFrameRate: Float) {
}
func setupAudioInput(configuration: MediaEditorVideoExport.Configuration) {
}
func startWriting() -> Bool {
if self.status != .failed {
self.status = .writing
return true
} else {
return false
}
}
func startSession(atSourceTime time: CMTime) {
}
func finishWriting(completion: @escaping () -> Void) {
self.ffmpegWriter.finalizeVideo()
self.status = .completed
completion()
}
func cancelWriting() {
}
func requestVideoDataWhenReady(on queue: DispatchQueue, using block: @escaping () -> Void) {
queue.async {
block()
}
}
func requestAudioDataWhenReady(on queue: DispatchQueue, using block: @escaping () -> Void) {
}
var isReadyForMoreVideoData: Bool {
return true
}
func appendVideoBuffer(_ buffer: CMSampleBuffer) -> Bool {
return false
}
func appendPixelBuffer(_ buffer: CVPixelBuffer, at time: CMTime) -> Bool {
let width = CVPixelBufferGetWidth(buffer)
let height = CVPixelBufferGetHeight(buffer)
let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer)
let frame = FFMpegAVFrame(pixelFormat: .YUVA, width: Int32(width), height: Int32(height))
CVPixelBufferLockBaseAddress(buffer, CVPixelBufferLockFlags.readOnly)
let src = CVPixelBufferGetBaseAddress(buffer)
var srcBuffer = vImage_Buffer(data: src, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: bytesPerRow)
var yBuffer = vImage_Buffer(data: frame.data[0], height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: width)
var uBuffer = vImage_Buffer(data: frame.data[1], height: vImagePixelCount(height / 2), width: vImagePixelCount(width / 2), rowBytes: width / 2)
var vBuffer = vImage_Buffer(data: frame.data[2], height: vImagePixelCount(height / 2), width: vImagePixelCount(width / 2), rowBytes: width / 2)
var aBuffer = vImage_Buffer(data: frame.data[3], height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: width)
var outInfo = self.conversionInfo
let _ = vImageConvert_ARGB8888To420Yp8_Cb8_Cr8(&srcBuffer, &yBuffer, &uBuffer, &vBuffer, &outInfo, [ 3, 2, 1, 0 ], vImage_Flags(kvImageDoNotTile))
vImageExtractChannel_ARGB8888(&srcBuffer, &aBuffer, 3, vImage_Flags(kvImageDoNotTile))
CVPixelBufferUnlockBaseAddress(buffer, CVPixelBufferLockFlags.readOnly)
return self.ffmpegWriter.encode(frame)
}
func markVideoAsFinished() {
}
var pixelBufferPool: CVPixelBufferPool? {
return self.pool
}
var isReadyForMoreAudioData: Bool {
return false
}
func appendAudioBuffer(_ buffer: CMSampleBuffer) -> Bool {
return false
}
func markAudioAsFinished() {
}
var status: ExportWriterStatus = .unknown
var error: Error?
}
@@ -0,0 +1,192 @@
import Foundation
import QuartzCore
import Metal
import simd
struct VertexData {
let pos: simd_float4
let texCoord: simd_float2
let localPos: simd_float2
}
public enum TextureRotation: Int {
case rotate0Degrees
case rotate0DegreesMirrored
case rotate90Degrees
case rotate180Degrees
case rotate270Degrees
case rotate90DegreesMirrored
}
func verticesDataForRotation(_ rotation: TextureRotation, rect: CGRect = CGRect(x: -0.5, y: -0.5, width: 1.0, height: 1.0), z: Float = 0.0) -> [VertexData] {
let topLeft: simd_float2
let topRight: simd_float2
let bottomLeft: simd_float2
let bottomRight: simd_float2
switch rotation {
case .rotate0Degrees:
topLeft = simd_float2(0.0, 1.0)
topRight = simd_float2(1.0, 1.0)
bottomLeft = simd_float2(0.0, 0.0)
bottomRight = simd_float2(1.0, 0.0)
case .rotate0DegreesMirrored:
topLeft = simd_float2(1.0, 1.0)
topRight = simd_float2(0.0, 1.0)
bottomLeft = simd_float2(1.0, 0.0)
bottomRight = simd_float2(0.0, 0.0)
case .rotate180Degrees:
topLeft = simd_float2(1.0, 0.0)
topRight = simd_float2(0.0, 0.0)
bottomLeft = simd_float2(1.0, 1.0)
bottomRight = simd_float2(0.0, 1.0)
case .rotate90Degrees:
topLeft = simd_float2(1.0, 1.0)
topRight = simd_float2(1.0, 0.0)
bottomLeft = simd_float2(0.0, 1.0)
bottomRight = simd_float2(0.0, 0.0)
case .rotate90DegreesMirrored:
topLeft = simd_float2(1.0, 0.0)
topRight = simd_float2(1.0, 1.0)
bottomLeft = simd_float2(0.0, 0.0)
bottomRight = simd_float2(0.0, 1.0)
case .rotate270Degrees:
topLeft = simd_float2(0.0, 0.0)
topRight = simd_float2(0.0, 1.0)
bottomLeft = simd_float2(1.0, 0.0)
bottomRight = simd_float2(1.0, 1.0)
}
return [
VertexData(
pos: simd_float4(x: Float(rect.minX) * 2.0, y: Float(rect.minY) * 2.0, z: z, w: 1),
texCoord: topLeft,
localPos: simd_float2(0.0, 0.0)
),
VertexData(
pos: simd_float4(x: Float(rect.maxX) * 2.0, y: Float(rect.minY) * 2.0, z: z, w: 1),
texCoord: topRight,
localPos: simd_float2(1.0, 0.0)
),
VertexData(
pos: simd_float4(x: Float(rect.minX) * 2.0, y: Float(rect.maxY) * 2.0, z: z, w: 1),
texCoord: bottomLeft,
localPos: simd_float2(0.0, 1.0)
),
VertexData(
pos: simd_float4(x: Float(rect.maxX) * 2.0, y: Float(rect.maxY) * 2.0, z: z, w: 1),
texCoord: bottomRight,
localPos: simd_float2(1.0, 1.0)
),
]
}
func textureDimensionsForRotation(texture: MTLTexture, rotation: TextureRotation) -> (width: Int, height: Int) {
switch rotation {
case .rotate90Degrees, .rotate90DegreesMirrored, .rotate270Degrees:
return (texture.height, texture.width)
default:
return (texture.width, texture.height)
}
}
class DefaultRenderPass: RenderPass {
fileprivate var pipelineState: MTLRenderPipelineState?
fileprivate var verticesBuffer: MTLBuffer?
fileprivate var textureRotation: TextureRotation = .rotate0Degrees
var vertexShaderFunctionName: String {
return "defaultVertexShader"
}
var fragmentShaderFunctionName: String {
return "defaultFragmentShader"
}
var pixelFormat: MTLPixelFormat {
return .bgra8Unorm
}
func setup(device: MTLDevice, library: MTLLibrary) {
let pipelineDescriptor = MTLRenderPipelineDescriptor()
pipelineDescriptor.vertexFunction = library.makeFunction(name: self.vertexShaderFunctionName)
pipelineDescriptor.fragmentFunction = library.makeFunction(name: self.fragmentShaderFunctionName)
pipelineDescriptor.colorAttachments[0].pixelFormat = self.pixelFormat
do {
self.pipelineState = try device.makeRenderPipelineState(descriptor: pipelineDescriptor)
} catch {
print(error.localizedDescription)
}
}
func setupVerticesBuffer(device: MTLDevice, rotation: TextureRotation = .rotate0Degrees) {
if self.verticesBuffer == nil || rotation != self.textureRotation {
self.textureRotation = rotation
let vertices = verticesDataForRotation(rotation)
self.verticesBuffer = device.makeBuffer(
bytes: vertices,
length: MemoryLayout<VertexData>.stride * vertices.count,
options: [])
}
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device)
return nil
}
func encodeDefaultCommands(using encoder: MTLRenderCommandEncoder) {
guard let pipelineState = self.pipelineState, let verticesBuffer = self.verticesBuffer else {
return
}
encoder.setRenderPipelineState(pipelineState)
encoder.setVertexBuffer(verticesBuffer, offset: 0, index: 0)
encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
}
}
final class OutputRenderPass: DefaultRenderPass {
weak var renderTarget: RenderTarget?
@discardableResult
override func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
guard let renderTarget = self.renderTarget else {
return nil
}
self.setupVerticesBuffer(device: device)
autoreleasepool {
guard let drawable = renderTarget.drawable else {
return
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = (drawable as? CAMetalDrawable)?.texture
renderPassDescriptor.colorAttachments[0].loadAction = .clear
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
let drawableSize = renderTarget.drawableSize
let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(
descriptor: renderPassDescriptor)!
renderCommandEncoder.setViewport(MTLViewport(
originX: 0.0, originY: 0.0,
width: Double(drawableSize.width), height: Double(drawableSize.height),
znear: -1.0, zfar: 1.0))
renderCommandEncoder.setFragmentTexture(input, index: 0)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
commandBuffer.present(drawable)
}
return nil
}
}
@@ -0,0 +1,17 @@
import Foundation
import Metal
import simd
final class SharpenRenderPass: RenderPass {
fileprivate var cachedTexture: MTLTexture?
var value: simd_float1 = 0.0
func setup(device: MTLDevice, library: MTLLibrary) {
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
return input
}
}
@@ -0,0 +1,137 @@
import Foundation
import UIKit
import Display
import Metal
import simd
import CoreImage
private let maxBorderWidth: Float = 40.0
final class StickerOutlineRenderPass: RenderPass {
var value: simd_float1 = 0.0
private var context: CIContext?
private var edgeMaskFilter: CIFilter?
private var edgeMaskImage: (CIImage, simd_float1)?
private var maskFilter: CIFilter?
private var alphaFilter: CIFilter?
private var blendFilter: CIFilter?
private var sourceAtopFilter: CIFilter?
private var whiteImage: CIImage?
private var outputTexture: MTLTexture?
func setup(device: MTLDevice, library: MTLLibrary) {
self.context = CIContext(mtlDevice: device, options: [.workingColorSpace : CGColorSpaceCreateDeviceRGB()])
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
guard self.value > 0.005, let context = self.context else {
return input
}
let width = self.value * maxBorderWidth
if self.maskFilter == nil {
self.edgeMaskFilter = CIFilter(name: "CIBlendWithMask")
self.edgeMaskFilter?.setValue(CIImage(color: .clear), forKey: kCIInputBackgroundImageKey)
self.maskFilter = CIFilter(name: "CIMorphologyMaximum")
self.alphaFilter = CIFilter(name: "CIColorMatrix")
self.alphaFilter?.setValue(CIVector(x: 0, y: 0, z: 0, w: 0), forKey: "inputRVector")
self.alphaFilter?.setValue(CIVector(x: 0, y: 0, z: 0, w: 0), forKey: "inputGVector")
self.alphaFilter?.setValue(CIVector(x: 0, y: 0, z: 0, w: 0), forKey: "inputBVector")
self.alphaFilter?.setValue(CIVector(x: 0, y: 0, z: 0, w: 1), forKey: "inputAVector")
self.blendFilter = CIFilter(name: "CIBlendWithMask")
self.sourceAtopFilter = CIFilter(name: "CISourceAtopCompositing")
self.whiteImage = CIImage(color: .white)
}
if self.edgeMaskImage == nil || self.edgeMaskImage?.1 != width {
self.edgeMaskImage = (roundedCornersMaskImage(outlineWidth: CGFloat(width)), width)
}
self.edgeMaskFilter?.setValue(CIImage(mtlTexture: input), forKey: kCIInputImageKey)
self.edgeMaskFilter?.setValue(self.edgeMaskImage?.0, forKey: kCIInputMaskImageKey)
guard let image = self.edgeMaskFilter?.outputImage else {
return input
}
guard let maskFilter = self.maskFilter else {
return input
}
maskFilter.setValue(width, forKey: kCIInputRadiusKey)
maskFilter.setValue(image, forKey: kCIInputImageKey)
guard let eroded = maskFilter.outputImage else {
return input
}
self.sourceAtopFilter?.setValue(self.whiteImage, forKey: kCIInputImageKey)
self.sourceAtopFilter?.setValue(eroded, forKey: kCIInputBackgroundImageKey)
guard let colorizedImage = self.sourceAtopFilter?.outputImage?.cropped(to: eroded.extent) else {
return input
}
self.alphaFilter?.setValue(image, forKey: kCIInputImageKey)
guard let alphaOnlyImage = self.alphaFilter?.outputImage, let whiteImage = self.whiteImage else {
return input
}
let blendMask = alphaOnlyImage.composited(over: whiteImage).cropped(to: alphaOnlyImage.extent)
self.blendFilter?.setValue(colorizedImage, forKey: kCIInputImageKey)
self.blendFilter?.setValue(blendMask, forKey: kCIInputMaskImageKey)
self.blendFilter?.setValue(CIImage(color: .clear), forKey: kCIInputBackgroundImageKey)
guard let outline = self.blendFilter?.outputImage else {
return input
}
var resultImage = outline.composited(over: image)
resultImage = outline.composited(over: resultImage)
if self.outputTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = input.width
textureDescriptor.height = input.height
textureDescriptor.pixelFormat = input.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return nil
}
self.outputTexture = texture
texture.label = "outlineOutputTexture"
}
guard let outputTexture = self.outputTexture else {
return input
}
let renderDestination = CIRenderDestination(mtlTexture: outputTexture, commandBuffer: commandBuffer)
_ = try? context.startTask(toRender: resultImage, to: renderDestination)
return outputTexture
}
}
private func roundedCornersMaskImage(outlineWidth: CGFloat) -> CIImage {
let rectSize = CGSize(width: floor(1080.0 * 0.97) - outlineWidth * 2.0, height: floor(1080.0 * 0.97) - outlineWidth * 2.0)
let cornerRadius = floor(1080.0 * 0.97) / 8.0 - outlineWidth
let image = generateImage(CGSize(width: 1080.0, height: 1920.0), opaque: true, scale: 1.0) { size, context in
context.setFillColor(UIColor.black.cgColor)
context.fill(CGRect(origin: .zero, size: size))
context.addPath(CGPath(roundedRect: CGRect(origin: CGPoint(x: floor((1080.0 - rectSize.width) / 2.0), y: floor((1920.0 - rectSize.width) / 2.0)), size: rectSize), cornerWidth: cornerRadius, cornerHeight: cornerRadius, transform: nil))
context.setFillColor(UIColor.white.cgColor)
context.fillPath()
}?.cgImage
return CIImage(cgImage: image!)
}
@@ -0,0 +1,374 @@
import Foundation
import AVFoundation
import Metal
import MetalKit
import ImageTransparency
import SwiftSignalKit
final class UniversalTextureSource: TextureSource {
enum Input {
case image(UIImage, CGRect?, CGFloat, CGPoint)
case video(AVPlayerItem, CGRect?, CGFloat, CGPoint)
case entity(MediaEditorComposerEntity)
fileprivate func createContext(renderTarget: RenderTarget, queue: DispatchQueue, additional: Bool) -> InputContext {
switch self {
case .image:
return ImageInputContext(input: self, renderTarget: renderTarget, queue: queue)
case .video:
return VideoInputContext(input: self, renderTarget: renderTarget, queue: queue, additional: additional)
case .entity:
return EntityInputContext(input: self, renderTarget: renderTarget, queue: queue)
}
}
}
private weak var renderTarget: RenderTarget?
private var displayLink: CADisplayLink?
private let queue: DispatchQueue
private var mainInputContext: InputContext?
private var additionalInputContexts: [InputContext] = []
var forceUpdates = false
private var rate: Float = 1.0
weak var output: MediaEditorRenderer?
init(renderTarget: RenderTarget) {
self.renderTarget = renderTarget
self.queue = DispatchQueue(
label: "UniversalTextureSource Queue",
qos: .userInteractive,
attributes: [],
autoreleaseFrequency: .workItem,
target: nil
)
}
var mainImage: UIImage? {
if let mainInput = self.mainInputContext?.input, case let .image(image, _, _, _) = mainInput {
return image
}
return nil
}
func setMainInput(_ input: Input) {
guard let renderTarget = self.renderTarget else {
return
}
self.mainInputContext = input.createContext(renderTarget: renderTarget, queue: self.queue, additional: false)
self.update(forced: true)
}
func setAdditionalInputs(_ inputs: [Input]) {
guard let renderTarget = self.renderTarget else {
return
}
self.additionalInputContexts = inputs.map { $0.createContext(renderTarget: renderTarget, queue: self.queue, additional: true) }
self.update(forced: true)
}
func setRate(_ rate: Float) {
self.rate = rate
}
private var previousAdditionalOutput: [Int: MediaEditorRenderer.Input] = [:]
private var readyForMoreData = Atomic<Bool>(value: true)
private func update(forced: Bool) {
let time = CACurrentMediaTime()
var fps: Int = 60
if self.mainInputContext?.useAsyncOutput == true {
fps = 30
}
var additionalsNeedDisplayLink = false
for context in self.additionalInputContexts {
if context.needsDisplayLink {
additionalsNeedDisplayLink = true
break
}
}
let needsDisplayLink = (self.mainInputContext?.needsDisplayLink ?? false) || additionalsNeedDisplayLink
if needsDisplayLink {
if self.displayLink == nil {
let displayLink = CADisplayLink(target: DisplayLinkTarget({ [weak self] in
if let self {
self.update(forced: self.forceUpdates)
}
}), selector: #selector(DisplayLinkTarget.handleDisplayLinkUpdate(sender:)))
displayLink.preferredFramesPerSecond = fps
displayLink.add(to: .main, forMode: .common)
self.displayLink = displayLink
}
} else {
if let displayLink = self.displayLink {
self.displayLink = nil
displayLink.invalidate()
}
}
guard self.rate > 0.0 || forced else {
return
}
if let mainInputContext = self.mainInputContext, mainInputContext.useAsyncOutput {
guard self.readyForMoreData.with({ $0 }) else {
return
}
let _ = self.readyForMoreData.swap(false)
mainInputContext.asyncOutput(time: time, completion: { [weak self] main in
guard let self else {
return
}
if let main {
self.output?.consume(main: main, additionals: [], render: true)
}
let _ = self.readyForMoreData.swap(true)
})
} else {
let main = self.mainInputContext?.output(time: time)
var additionals: [(Int, InputContext.Output?)] = []
var index = 0
for context in self.additionalInputContexts {
additionals.append((index, context.output(time: time)))
index += 1
}
for (index, output) in additionals {
if let output {
self.previousAdditionalOutput[index] = output
}
}
for (index, output) in additionals {
if output == nil {
additionals[index] = (index, self.previousAdditionalOutput[index])
}
}
guard let main else {
return
}
self.output?.consume(main: main, additionals: additionals.compactMap { $0.1 }, render: true)
}
}
func connect(to consumer: MediaEditorRenderer) {
self.output = consumer
self.update(forced: true)
}
func invalidate() {
self.mainInputContext?.invalidate()
self.additionalInputContexts.forEach { $0.invalidate() }
}
private class DisplayLinkTarget {
private let update: () -> Void
init(_ update: @escaping () -> Void) {
self.update = update
}
@objc func handleDisplayLinkUpdate(sender: CADisplayLink) {
self.update()
}
}
}
protocol InputContext {
typealias Input = UniversalTextureSource.Input
typealias Output = MediaEditorRenderer.Input
var input: Input { get }
var rect: CGRect? { get }
var useAsyncOutput: Bool { get }
func output(time: Double) -> Output?
func asyncOutput(time: Double, completion: @escaping (Output?) -> Void)
var needsDisplayLink: Bool { get }
func invalidate()
}
extension InputContext {
var useAsyncOutput: Bool {
return false
}
func asyncOutput(time: Double, completion: @escaping (Output?) -> Void) {
completion(self.output(time: time))
}
}
private class ImageInputContext: InputContext {
fileprivate var input: Input
private var texture: MTLTexture?
private var hasTransparency = false
fileprivate var rect: CGRect?
fileprivate var scale: CGFloat
fileprivate var offset: CGPoint
init(input: Input, renderTarget: RenderTarget, queue: DispatchQueue) {
guard case let .image(image, rect, scale, offset) = input else {
fatalError()
}
self.input = input
self.rect = rect
self.scale = scale
self.offset = offset
if let device = renderTarget.mtlDevice {
self.texture = loadTexture(image: image, device: device)
}
self.hasTransparency = imageHasTransparency(image)
}
func output(time: Double) -> Output? {
return self.texture.flatMap { .texture($0, .zero, self.hasTransparency, self.rect, self.scale, self.offset) }
}
func invalidate() {
self.texture = nil
}
var needsDisplayLink: Bool {
return false
}
}
private class VideoInputContext: NSObject, InputContext, AVPlayerItemOutputPullDelegate {
fileprivate var input: Input
private var videoOutput: AVPlayerItemVideoOutput?
private var textureRotation: TextureRotation = .rotate0Degrees
fileprivate var rect: CGRect?
fileprivate var scale: CGFloat
fileprivate var offset: CGPoint
var playerItem: AVPlayerItem {
guard case let .video(playerItem, _, _, _) = self.input else {
fatalError()
}
return playerItem
}
init(input: Input, renderTarget: RenderTarget, queue: DispatchQueue, additional: Bool) {
guard case let .video(_, rect, scale, offset) = input else {
fatalError()
}
self.input = input
self.rect = rect
self.scale = scale
self.offset = offset
super.init()
//TODO: mirror if self.additionalPlayer == nil && self.mirror
self.textureRotation = textureRotatonForAVAsset(self.playerItem.asset, mirror: rect == nil ? additional : false)
let colorProperties: [String: Any] = [
AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2,
AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2,
AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2
]
let outputSettings: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
kCVPixelBufferMetalCompatibilityKey as String: true,
AVVideoColorPropertiesKey: colorProperties
]
let videoOutput = AVPlayerItemVideoOutput(outputSettings: outputSettings)
videoOutput.suppressesPlayerRendering = true
videoOutput.setDelegate(self, queue: queue)
self.playerItem.add(videoOutput)
self.videoOutput = videoOutput
}
func output(time: Double) -> Output? {
guard let videoOutput = self.videoOutput else {
return nil
}
let requestTime = videoOutput.itemTime(forHostTime: time)
if requestTime < .zero {
return nil
}
var presentationTime: CMTime = .zero
var videoPixelBuffer: VideoPixelBuffer?
if let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: requestTime, itemTimeForDisplay: &presentationTime) {
videoPixelBuffer = VideoPixelBuffer(pixelBuffer: pixelBuffer, rotation: self.textureRotation, timestamp: presentationTime)
}
return videoPixelBuffer.flatMap { .videoBuffer($0, self.rect, self.scale, self.offset) }
}
func invalidate() {
if let videoOutput = self.videoOutput {
self.videoOutput = nil
self.playerItem.remove(videoOutput)
videoOutput.setDelegate(nil, queue: nil)
}
}
var needsDisplayLink: Bool {
return true
}
}
final class EntityInputContext: NSObject, InputContext, AVPlayerItemOutputPullDelegate {
internal var input: Input
private var textureRotation: TextureRotation = .rotate0Degrees
var rect: CGRect?
var entity: MediaEditorComposerEntity {
guard case let .entity(entity) = self.input else {
fatalError()
}
return entity
}
private let ciContext: CIContext
private let startTime: Double
init(input: Input, renderTarget: RenderTarget, queue: DispatchQueue) {
guard case .entity = input else {
fatalError()
}
self.input = input
self.ciContext = CIContext(options: [.workingColorSpace : CGColorSpaceCreateDeviceRGB()])
self.startTime = CACurrentMediaTime()
super.init()
self.textureRotation = .rotate0Degrees
}
func output(time: Double) -> Output? {
return nil
}
func asyncOutput(time: Double, completion: @escaping (Output?) -> Void) {
let deltaTime = max(0.0, time - self.startTime)
let timestamp = CMTime(seconds: deltaTime, preferredTimescale: CMTimeScale(NSEC_PER_SEC))
self.entity.image(for: timestamp, frameRate: 30, context: self.ciContext, completion: { image in
Queue.mainQueue().async {
completion(image.flatMap { .ciImage($0, timestamp) })
}
})
}
func invalidate() {
}
var needsDisplayLink: Bool {
if let entity = self.entity as? MediaEditorComposerStickerEntity, entity.isAnimated {
return true
}
return false
}
var useAsyncOutput: Bool {
return true
}
}
@@ -0,0 +1,905 @@
import Foundation
import AVFoundation
import Metal
import MetalKit
import SwiftSignalKit
private func verticesData(
textureRotation: TextureRotation,
containerSize: CGSize,
position: CGPoint,
size: CGSize,
rotation: CGFloat,
mirror: Bool = false,
z: Float = 0.0
) -> [VertexData] {
var topLeft: simd_float2
var topRight: simd_float2
var bottomLeft: simd_float2
var bottomRight: simd_float2
switch textureRotation {
case .rotate0Degrees:
topLeft = simd_float2(0.0, 1.0)
topRight = simd_float2(1.0, 1.0)
bottomLeft = simd_float2(0.0, 0.0)
bottomRight = simd_float2(1.0, 0.0)
case .rotate0DegreesMirrored:
topLeft = simd_float2(1.0, 1.0)
topRight = simd_float2(0.0, 1.0)
bottomLeft = simd_float2(1.0, 0.0)
bottomRight = simd_float2(0.0, 0.0)
case .rotate180Degrees:
topLeft = simd_float2(1.0, 0.0)
topRight = simd_float2(0.0, 0.0)
bottomLeft = simd_float2(1.0, 1.0)
bottomRight = simd_float2(0.0, 1.0)
case .rotate90Degrees:
topLeft = simd_float2(1.0, 1.0)
topRight = simd_float2(1.0, 0.0)
bottomLeft = simd_float2(0.0, 1.0)
bottomRight = simd_float2(0.0, 0.0)
case .rotate90DegreesMirrored:
topLeft = simd_float2(1.0, 0.0)
topRight = simd_float2(1.0, 1.0)
bottomLeft = simd_float2(0.0, 0.0)
bottomRight = simd_float2(0.0, 1.0)
case .rotate270Degrees:
topLeft = simd_float2(0.0, 0.0)
topRight = simd_float2(0.0, 1.0)
bottomLeft = simd_float2(1.0, 0.0)
bottomRight = simd_float2(1.0, 1.0)
}
if mirror {
topLeft = simd_float2(1.0 - topLeft.x, topLeft.y)
topRight = simd_float2(1.0 - topRight.x, topRight.y)
bottomLeft = simd_float2(1.0 - bottomLeft.x, bottomLeft.y)
bottomRight = simd_float2(1.0 - bottomRight.x, bottomRight.y)
}
let containerSize = CGSize(width: containerSize.width, height: containerSize.height)
let angle = Float(.pi - rotation)
let cosAngle = cos(angle)
let sinAngle = sin(angle)
let centerX = Float(position.x)
let centerY = Float(position.y)
let halfWidth = Float(size.width / 2.0)
let halfHeight = Float(size.height / 2.0)
return [
VertexData(
pos: simd_float4(
x: (centerX + (halfWidth * cosAngle) - (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY + (halfWidth * sinAngle) + (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: topLeft,
localPos: simd_float2(0.0, 0.0)
),
VertexData(
pos: simd_float4(
x: (centerX - (halfWidth * cosAngle) - (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY - (halfWidth * sinAngle) + (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: topRight,
localPos: simd_float2(1.0, 0.0)
),
VertexData(
pos: simd_float4(
x: (centerX + (halfWidth * cosAngle) + (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY + (halfWidth * sinAngle) - (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: bottomLeft,
localPos: simd_float2(0.0, 1.0)
),
VertexData(
pos: simd_float4(
x: (centerX - (halfWidth * cosAngle) + (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY - (halfWidth * sinAngle) - (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: bottomRight,
localPos: simd_float2(1.0, 1.0)
)
]
}
private func verticesData(
size: CGSize,
textureRotation: TextureRotation,
containerSize: CGSize,
textureRect: CGRect,
scale: simd_float1,
offset: simd_float2,
z: Float = 0.0
) -> [VertexData] {
let textureRect = CGRect(origin: CGPoint(x: textureRect.origin.x, y: containerSize.height - textureRect.maxY ), size: textureRect.size)
let containerAspect = textureRect.width / textureRect.height
let imageAspect = size.width / size.height
var texCoordScale: simd_float2
if imageAspect > containerAspect {
texCoordScale = simd_float2(Float(containerAspect / imageAspect), 1.0)
} else {
texCoordScale = simd_float2(1.0, Float(imageAspect / containerAspect))
}
let adjustedOffset = simd_float2(
offset.x / texCoordScale.x,
offset.y / texCoordScale.y
)
texCoordScale *= 1.0 / scale
let scaledTopLeft = simd_float2(0.5 - texCoordScale.x * 0.5, 0.5 + texCoordScale.y * 0.5) - adjustedOffset
let scaledTopRight = simd_float2(0.5 + texCoordScale.x * 0.5, 0.5 + texCoordScale.y * 0.5) - adjustedOffset
let scaledBottomLeft = simd_float2(0.5 - texCoordScale.x * 0.5, 0.5 - texCoordScale.y * 0.5) - adjustedOffset
let scaledBottomRight = simd_float2(0.5 + texCoordScale.x * 0.5, 0.5 - texCoordScale.y * 0.5) - adjustedOffset
let topLeft: simd_float2
let topRight: simd_float2
let bottomLeft: simd_float2
let bottomRight: simd_float2
switch textureRotation {
case .rotate0Degrees:
topLeft = scaledTopLeft
topRight = scaledTopRight
bottomLeft = scaledBottomLeft
bottomRight = scaledBottomRight
case .rotate0DegreesMirrored:
topLeft = scaledTopRight
topRight = scaledTopLeft
bottomLeft = scaledBottomRight
bottomRight = scaledBottomLeft
case .rotate180Degrees:
topLeft = scaledBottomRight
topRight = scaledBottomLeft
bottomLeft = scaledTopRight
bottomRight = scaledTopLeft
case .rotate90Degrees:
topLeft = scaledTopRight
topRight = scaledBottomRight
bottomLeft = scaledTopLeft
bottomRight = scaledBottomLeft
case .rotate90DegreesMirrored:
topLeft = scaledBottomRight
topRight = scaledTopRight
bottomLeft = scaledBottomLeft
bottomRight = scaledTopLeft
case .rotate270Degrees:
topLeft = scaledBottomLeft
topRight = scaledTopLeft
bottomLeft = scaledBottomRight
bottomRight = scaledTopRight
}
let containerSize = CGSize(width: containerSize.width, height: containerSize.height)
let centerX = Float(textureRect.midX - containerSize.width / 2.0)
let centerY = Float(textureRect.midY - containerSize.height / 2.0)
let halfWidth = Float(textureRect.width / 2.0)
let halfHeight = Float(textureRect.height / 2.0)
let angle = Float.pi
let cosAngle = cos(angle)
let sinAngle = sin(angle)
return [
VertexData(
pos: simd_float4(
x: (centerX + (halfWidth * cosAngle) - (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY + (halfWidth * sinAngle) + (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: topLeft,
localPos: simd_float2(0.0, 0.0)
),
VertexData(
pos: simd_float4(
x: (centerX - (halfWidth * cosAngle) - (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY - (halfWidth * sinAngle) + (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: topRight,
localPos: simd_float2(1.0, 0.0)
),
VertexData(
pos: simd_float4(
x: (centerX + (halfWidth * cosAngle) + (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY + (halfWidth * sinAngle) - (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: bottomLeft,
localPos: simd_float2(0.0, 1.0)
),
VertexData(
pos: simd_float4(
x: (centerX - (halfWidth * cosAngle) + (halfHeight * sinAngle)) / Float(containerSize.width) * 2.0,
y: (centerY - (halfWidth * sinAngle) - (halfHeight * cosAngle)) / Float(containerSize.height) * 2.0,
z: z,
w: 1
),
texCoord: bottomRight,
localPos: simd_float2(1.0, 1.0)
)
]
}
private func lookupSpringValue(_ t: CGFloat) -> CGFloat {
let table: [(CGFloat, CGFloat)] = [
(0.0, 0.0),
(0.0625, 0.1123005598783493),
(0.125, 0.31598418951034546),
(0.1875, 0.5103585720062256),
(0.25, 0.6650152802467346),
(0.3125, 0.777747631072998),
(0.375, 0.8557760119438171),
(0.4375, 0.9079672694206238),
(0.5, 0.942038357257843),
(0.5625, 0.9638798832893372),
(0.625, 0.9776856303215027),
(0.6875, 0.9863143563270569),
(0.75, 0.991658091545105),
(0.8125, 0.9949421286582947),
(0.875, 0.9969474077224731),
(0.9375, 0.9981651306152344),
(1.0, 1.0)
]
for i in 0 ..< table.count - 2 {
let lhs = table[i]
let rhs = table[i + 1]
if t >= lhs.0 && t <= rhs.0 {
let fraction = (t - lhs.0) / (rhs.0 - lhs.0)
let value = lhs.1 + fraction * (rhs.1 - lhs.1)
return value
}
}
return 1.0
}
private var transitionDuration = 0.5
private var apperanceDuration = 0.2
private var videoRemovalDuration: Double = 0.2
struct VideoEncodeParameters {
var dimensions: simd_float2
var roundness: simd_float1
var alpha: simd_float1
var isOpaque: simd_float1
var empty: simd_float1 = 0.0
}
final class VideoFinishPass: RenderPass {
private var cachedTexture: MTLTexture?
var gradientPipelineState: MTLRenderPipelineState?
var mainPipelineState: MTLRenderPipelineState?
var mainTextureRotation: TextureRotation = .rotate0Degrees
var additionalTextureRotation: TextureRotation = .rotate0Degrees
var pixelFormat: MTLPixelFormat {
return .bgra8Unorm
}
func setup(device: MTLDevice, library: MTLLibrary) {
let mainDescriptor = MTLRenderPipelineDescriptor()
mainDescriptor.vertexFunction = library.makeFunction(name: "defaultVertexShader")
mainDescriptor.fragmentFunction = library.makeFunction(name: "dualFragmentShader")
mainDescriptor.colorAttachments[0].pixelFormat = self.pixelFormat
mainDescriptor.colorAttachments[0].isBlendingEnabled = true
mainDescriptor.colorAttachments[0].rgbBlendOperation = .add
mainDescriptor.colorAttachments[0].alphaBlendOperation = .add
mainDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha
mainDescriptor.colorAttachments[0].sourceAlphaBlendFactor = .sourceAlpha
mainDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha
mainDescriptor.colorAttachments[0].destinationAlphaBlendFactor = .oneMinusSourceAlpha
let gradientDescriptor = MTLRenderPipelineDescriptor()
gradientDescriptor.vertexFunction = library.makeFunction(name: "defaultVertexShader")
gradientDescriptor.fragmentFunction = library.makeFunction(name: "gradientFragmentShader")
gradientDescriptor.colorAttachments[0].pixelFormat = self.pixelFormat
gradientDescriptor.colorAttachments[0].isBlendingEnabled = true
gradientDescriptor.colorAttachments[0].rgbBlendOperation = .add
gradientDescriptor.colorAttachments[0].alphaBlendOperation = .add
gradientDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha
gradientDescriptor.colorAttachments[0].sourceAlphaBlendFactor = .sourceAlpha
gradientDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha
gradientDescriptor.colorAttachments[0].destinationAlphaBlendFactor = .oneMinusSourceAlpha
do {
self.mainPipelineState = try device.makeRenderPipelineState(descriptor: mainDescriptor)
self.gradientPipelineState = try device.makeRenderPipelineState(descriptor: gradientDescriptor)
} catch {
print(error.localizedDescription)
}
}
func encodeVideo(
using encoder: MTLRenderCommandEncoder,
containerSize: CGSize,
texture: MTLTexture,
textureRotation: TextureRotation,
rect: CGRect,
scale: CGFloat,
offset: CGPoint,
zPosition: Float,
device: MTLDevice
) {
encoder.setFragmentTexture(texture, index: 0)
encoder.setFragmentTexture(texture, index: 1)
let vertices = verticesData(
size: CGSize(width: texture.width, height: texture.height),
textureRotation: textureRotation,
containerSize: containerSize,
textureRect: rect,
scale: simd_float1(scale),
offset: simd_float2(Float(offset.x / scale), Float(-offset.y / scale)),
z: zPosition
)
let buffer = device.makeBuffer(
bytes: vertices,
length: MemoryLayout<VertexData>.stride * vertices.count,
options: [])
encoder.setVertexBuffer(buffer, offset: 0, index: 0)
var parameters = VideoEncodeParameters(
dimensions: simd_float2(Float(rect.size.width), Float(rect.size.height)),
roundness: 0.0,
alpha: 1.0,
isOpaque: 1.0
)
encoder.setFragmentBytes(&parameters, length: MemoryLayout<VideoEncodeParameters>.size, index: 0)
encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
}
func encodeVideo(
using encoder: MTLRenderCommandEncoder,
containerSize: CGSize,
texture: MTLTexture,
textureRotation: TextureRotation,
maskTexture: MTLTexture?,
hasTransparency: Bool,
position: VideoPosition,
roundness: Float,
alpha: Float,
zPosition: Float,
device: MTLDevice
) {
encoder.setFragmentTexture(texture, index: 0)
if let maskTexture {
encoder.setFragmentTexture(maskTexture, index: 1)
} else {
encoder.setFragmentTexture(texture, index: 1)
}
let center = CGPoint(
x: position.position.x - containerSize.width / 2.0,
y: containerSize.height - position.position.y - containerSize.height / 2.0
)
let size = CGSize(
width: position.size.width * position.scale * position.baseScale,
height: position.size.height * position.scale * position.baseScale
)
let vertices = verticesData(
textureRotation: textureRotation,
containerSize: containerSize,
position: center,
size: size,
rotation: position.rotation,
mirror: position.mirroring,
z: zPosition
)
let buffer = device.makeBuffer(
bytes: vertices,
length: MemoryLayout<VertexData>.stride * vertices.count,
options: [])
encoder.setVertexBuffer(buffer, offset: 0, index: 0)
var parameters = VideoEncodeParameters(
dimensions: simd_float2(Float(size.width), Float(size.height)),
roundness: roundness,
alpha: alpha,
isOpaque: maskTexture == nil ? 1.0 : 0.0
)
encoder.setFragmentBytes(&parameters, length: MemoryLayout<VideoEncodeParameters>.size, index: 0)
encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
}
private let canvasSize = CGSize(width: 1080.0, height: 1920.0)
private var gradientColors = GradientColors(topColor: simd_float4(0.0, 0.0, 0.0, 0.0), bottomColor: simd_float4(0.0, 0.0, 0.0, 0.0))
func update(values: MediaEditorValues, videoDuration: Double?, additionalVideoDuration: Double?) {
let position = CGPoint(
x: canvasSize.width / 2.0 + values.cropOffset.x,
y: canvasSize.height / 2.0 + values.cropOffset.y
)
self.isStory = values.isStory || values.isSticker || values.isAvatar || values.isCover
self.isSticker = values.gradientColors?.first?.alpha == 0.0
self.coverDimensions = values.coverDimensions
self.mainPosition = VideoFinishPass.VideoPosition(position: position, size: self.mainPosition.size, scale: values.cropScale, rotation: values.cropRotation, mirroring: values.cropMirroring, baseScale: self.mainPosition.baseScale)
if let position = values.additionalVideoPosition, let scale = values.additionalVideoScale, let rotation = values.additionalVideoRotation {
self.additionalPosition = VideoFinishPass.VideoPosition(position: position, size: CGSize(width: 1080.0 / 4.0, height: 1440.0 / 4.0), scale: scale, rotation: rotation, mirroring: false, baseScale: self.additionalPosition.baseScale)
}
if !values.additionalVideoPositionChanges.isEmpty {
self.videoPositionChanges = values.additionalVideoPositionChanges
}
self.videoDuration = videoDuration
self.additionalVideoDuration = additionalVideoDuration
self.videoRange = values.videoTrimRange
self.additionalVideoRange = values.additionalVideoTrimRange
self.additionalVideoOffset = values.additionalVideoOffset
if let gradientColors = values.gradientColors, let top = gradientColors.first, let bottom = gradientColors.last {
let (topRed, topGreen, topBlue, topAlpha) = top.components
let (bottomRed, bottomGreen, bottomBlue, bottomAlpha) = bottom.components
self.gradientColors = GradientColors(
topColor: simd_float4(Float(topRed), Float(topGreen), Float(topBlue), Float(topAlpha)),
bottomColor: simd_float4(Float(bottomRed), Float(bottomGreen), Float(bottomBlue), Float(bottomAlpha))
)
}
}
private var mainPosition = VideoPosition(
position: CGPoint(x: 1080 / 2.0, y: 1920.0 / 2.0),
size: CGSize(width: 1080.0, height: 1920.0),
scale: 1.0,
rotation: 0.0,
mirroring: false,
baseScale: 1.0
)
private var additionalPosition = VideoPosition(
position: CGPoint(x: 1080 / 2.0, y: 1920.0 / 2.0),
size: CGSize(width: 1440.0, height: 1920.0),
scale: 0.5,
rotation: 0.0,
mirroring: false,
baseScale: 1.0
)
private var isStory = true
private var isSticker = true
private var coverDimensions: CGSize?
private var videoPositionChanges: [VideoPositionChange] = []
private var videoDuration: Double?
private var additionalVideoDuration: Double?
private var videoRange: Range<Double>?
private var additionalVideoRange: Range<Double>?
private var additionalVideoOffset: Double?
enum VideoType {
case main
case additional
case transition
}
struct VideoPosition {
let position: CGPoint
let size: CGSize
let scale: CGFloat
let rotation: CGFloat
let mirroring: Bool
let baseScale: CGFloat
func with(position: CGPoint) -> VideoPosition {
return VideoPosition(position: position, size: self.size, scale: self.scale, rotation: self.rotation, mirroring: self.mirroring, baseScale: baseScale)
}
func with(size: CGSize, baseScale: CGFloat) -> VideoPosition {
return VideoPosition(position: self.position, size: size, scale: self.scale, rotation: self.rotation, mirroring: self.mirroring, baseScale: baseScale)
}
func mixed(with other: VideoPosition, fraction: CGFloat) -> VideoPosition {
let position = CGPoint(
x: self.position.x + (other.position.x - self.position.x) * fraction,
y: self.position.y + (other.position.y - self.position.y) * fraction
)
let size = CGSize(
width: self.size.width + (other.size.width - self.size.width) * fraction,
height: self.size.height + (other.size.height - self.size.height) * fraction
)
let scale = self.scale + (other.scale - self.scale) * fraction
let rotation = self.rotation + (other.rotation - self.rotation) * fraction
return VideoPosition(
position: position,
size: size,
scale: scale,
rotation: rotation,
mirroring: self.mirroring,
baseScale: self.baseScale
)
}
}
struct VideoState {
let texture: MTLTexture
let textureRotation: TextureRotation
let position: VideoPosition
let roundness: Float
let alpha: Float
}
private var additionalVideoRemovalStartTimestamp: Double?
func animateAdditionalRemoval(completion: @escaping () -> Void) {
self.additionalVideoRemovalStartTimestamp = CACurrentMediaTime()
Queue.mainQueue().after(videoRemovalDuration) {
completion()
self.additionalVideoRemovalStartTimestamp = nil
}
}
func transitionState(for time: CMTime, mainInput: MTLTexture, additionalInput: MTLTexture?) -> (VideoState, VideoState?, VideoState?) {
let timestamp = time.seconds
var backgroundTexture = mainInput
var backgroundTextureRotation = self.mainTextureRotation
var foregroundTexture = additionalInput
var foregroundTextureRotation = self.additionalTextureRotation
var mainPosition = self.mainPosition
var additionalPosition = self.additionalPosition
var disappearingPosition = self.mainPosition
var transitionFraction = 1.0
if let additionalInput {
var previousChange: VideoPositionChange?
for change in self.videoPositionChanges {
if let _ = change.translationFrom {
continue
}
if timestamp >= change.timestamp {
previousChange = change
}
if timestamp < change.timestamp {
break
}
}
if let previousChange {
if previousChange.additional {
backgroundTexture = additionalInput
backgroundTextureRotation = self.additionalTextureRotation
mainPosition = VideoPosition(position: mainPosition.position, size: CGSize(width: 1440.0, height: 1920.0), scale: mainPosition.scale, rotation: mainPosition.rotation, mirroring: mainPosition.mirroring, baseScale: mainPosition.baseScale)
additionalPosition = VideoPosition(position: additionalPosition.position, size: CGSize(width: 1080.0 / 4.0, height: 1920.0 / 4.0), scale: additionalPosition.scale, rotation: additionalPosition.rotation, mirroring: additionalPosition.mirroring, baseScale: additionalPosition.baseScale)
foregroundTexture = mainInput
foregroundTextureRotation = self.mainTextureRotation
} else {
disappearingPosition = VideoPosition(position: mainPosition.position, size: CGSize(width: 1440.0, height: 1920.0), scale: mainPosition.scale, rotation: mainPosition.rotation, mirroring: mainPosition.mirroring, baseScale: mainPosition.baseScale)
}
if previousChange.timestamp > 0.0 && timestamp < previousChange.timestamp + transitionDuration {
transitionFraction = (timestamp - previousChange.timestamp) / transitionDuration
}
}
}
var translationTransitionFraction = 1.0
var previousAdditionalPosition = additionalPosition
if let _ = additionalInput {
var previousChange: VideoPositionChange?
for change in self.videoPositionChanges {
guard let _ = change.translationFrom else {
continue
}
if timestamp >= change.timestamp {
previousChange = change
}
if timestamp < change.timestamp {
break
}
}
if let previousChange, let translationFrom = previousChange.translationFrom {
previousAdditionalPosition = previousAdditionalPosition.with(position: translationFrom)
if previousChange.timestamp > 0.0 && timestamp < previousChange.timestamp + transitionDuration {
translationTransitionFraction = (timestamp - previousChange.timestamp) / transitionDuration
}
}
}
var backgroundVideoState = VideoState(texture: backgroundTexture, textureRotation: backgroundTextureRotation, position: mainPosition, roundness: 0.0, alpha: 1.0)
var foregroundVideoState: VideoState?
var disappearingVideoState: VideoState?
if let foregroundTexture {
var foregroundPosition = additionalPosition
var foregroundAlpha: Float = 1.0
if transitionFraction < 1.0 {
let springFraction = lookupSpringValue(transitionFraction)
let appearingPosition = VideoPosition(position: additionalPosition.position, size: additionalPosition.size, scale: 0.01, rotation: self.additionalPosition.rotation, mirroring: self.additionalPosition.mirroring, baseScale: self.additionalPosition.baseScale)
let backgroundInitialPosition = VideoPosition(position: additionalPosition.position, size: CGSize(width: mainPosition.size.width / 4.0, height: mainPosition.size.height / 4.0), scale: additionalPosition.scale, rotation: additionalPosition.rotation, mirroring: additionalPosition.mirroring, baseScale: additionalPosition.baseScale)
foregroundPosition = appearingPosition.mixed(with: additionalPosition, fraction: springFraction)
disappearingVideoState = VideoState(texture: foregroundTexture, textureRotation: foregroundTextureRotation, position: disappearingPosition, roundness: 0.0, alpha: 1.0)
backgroundVideoState = VideoState(texture: backgroundTexture, textureRotation: backgroundTextureRotation, position: backgroundInitialPosition.mixed(with: mainPosition, fraction: springFraction), roundness: Float(1.0 - springFraction), alpha: 1.0)
foregroundAlpha = min(1.0, max(0.0, Float(transitionFraction) * 2.5))
}
if translationTransitionFraction < 1.0 {
let springFraction = lookupSpringValue(translationTransitionFraction)
foregroundPosition = previousAdditionalPosition.mixed(with: foregroundPosition, fraction: springFraction)
}
var isVisible = true
var trimRangeLowerBound: Double?
var trimRangeUpperBound: Double?
if let additionalVideoRange = self.additionalVideoRange {
if let additionalVideoOffset = self.additionalVideoOffset {
trimRangeLowerBound = additionalVideoRange.lowerBound - additionalVideoOffset
trimRangeUpperBound = additionalVideoRange.upperBound - additionalVideoOffset
} else {
trimRangeLowerBound = additionalVideoRange.lowerBound
trimRangeUpperBound = additionalVideoRange.upperBound
}
} else if let additionalVideoOffset = self.additionalVideoOffset {
trimRangeLowerBound = -additionalVideoOffset
if let additionalVideoDuration = self.additionalVideoDuration {
trimRangeUpperBound = -additionalVideoOffset + additionalVideoDuration
}
}
if (trimRangeLowerBound != nil || trimRangeUpperBound != nil), let _ = self.videoDuration {
let disappearingPosition = VideoPosition(position: foregroundPosition.position, size: foregroundPosition.size, scale: 0.01, rotation: foregroundPosition.rotation, mirroring: foregroundPosition.mirroring, baseScale: foregroundPosition.baseScale)
let mainLowerBound = self.videoRange?.lowerBound ?? 0.0
if let trimRangeLowerBound, trimRangeLowerBound > mainLowerBound + 0.1, timestamp < trimRangeLowerBound + apperanceDuration {
let visibilityFraction = max(0.0, min(1.0, (timestamp - trimRangeLowerBound) / apperanceDuration))
if visibilityFraction.isZero {
isVisible = false
}
foregroundAlpha = Float(visibilityFraction)
foregroundPosition = disappearingPosition.mixed(with: foregroundPosition, fraction: visibilityFraction)
} else if let trimRangeUpperBound, timestamp > trimRangeUpperBound - apperanceDuration {
let visibilityFraction = 1.0 - max(0.0, min(1.0, (timestamp - trimRangeUpperBound) / apperanceDuration))
if visibilityFraction.isZero {
isVisible = false
}
foregroundAlpha = Float(visibilityFraction)
foregroundPosition = disappearingPosition.mixed(with: foregroundPosition, fraction: visibilityFraction)
}
}
if isVisible {
if let additionalVideoRemovalStartTimestamp {
let disappearingPosition = VideoPosition(position: foregroundPosition.position, size: foregroundPosition.size, scale: 0.01, rotation: foregroundPosition.rotation, mirroring: foregroundPosition.mirroring, baseScale: foregroundPosition.baseScale)
let visibilityFraction = max(0.0, min(1.0, 1.0 - (CACurrentMediaTime() - additionalVideoRemovalStartTimestamp) / videoRemovalDuration))
if visibilityFraction.isZero {
isVisible = false
}
foregroundAlpha = Float(visibilityFraction)
foregroundPosition = disappearingPosition.mixed(with: foregroundPosition, fraction: visibilityFraction)
}
foregroundVideoState = VideoState(texture: foregroundTexture, textureRotation: foregroundTextureRotation, position: foregroundPosition, roundness: 1.0, alpha: foregroundAlpha)
}
}
return (backgroundVideoState, foregroundVideoState, disappearingVideoState)
}
struct Input {
let texture: MTLTexture
let hasTransparency: Bool
let rect: CGRect?
let scale: CGFloat
let offset: CGPoint
}
func process(
input: Input,
inputMask: MTLTexture?,
hasTransparency: Bool,
secondInput: [Input],
timestamp: CMTime,
device: MTLDevice,
commandBuffer: MTLCommandBuffer
) -> MTLTexture? {
if !self.isStory {
return input.texture
}
let baseScale: CGFloat
if let dimensions = self.coverDimensions {
let fittedCanvasDimensions = dimensions.aspectFitted(canvasSize)
baseScale = max(fittedCanvasDimensions.width / CGFloat(input.texture.width), fittedCanvasDimensions.height / CGFloat(input.texture.height))
} else if !self.isSticker {
if input.texture.height > input.texture.width {
baseScale = max(canvasSize.width / CGFloat(input.texture.width), canvasSize.height / CGFloat(input.texture.height))
} else {
baseScale = canvasSize.width / CGFloat(input.texture.width)
}
} else {
if input.texture.height > input.texture.width {
baseScale = canvasSize.width / CGFloat(input.texture.width)
} else {
baseScale = canvasSize.width / CGFloat(input.texture.height)
}
}
self.mainPosition = self.mainPosition.with(size: CGSize(width: input.texture.width, height: input.texture.height), baseScale: baseScale)
let containerSize = canvasSize
if self.cachedTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = Int(containerSize.width)
textureDescriptor.height = Int(containerSize.height)
textureDescriptor.pixelFormat = input.texture.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return input.texture
}
self.cachedTexture = texture
texture.label = "finishedTexture"
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
if self.gradientColors.topColor.w > 0.0 {
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
} else {
renderPassDescriptor.colorAttachments[0].loadAction = .clear
}
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return input.texture
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(containerSize.width), height: Double(containerSize.height),
znear: -1.0, zfar: 1.0)
)
if self.gradientColors.topColor.w > 0.0 {
self.encodeGradient(
using: renderCommandEncoder,
containerSize: containerSize,
device: device
)
}
renderCommandEncoder.setRenderPipelineState(self.mainPipelineState!)
if let rect = input.rect {
self.encodeVideo(
using: renderCommandEncoder,
containerSize: containerSize,
texture: input.texture,
textureRotation: self.mainTextureRotation,
rect: rect,
scale: input.scale,
offset: input.offset,
zPosition: 0.0,
device: device
)
for input in secondInput {
if let rect = input.rect {
self.encodeVideo(
using: renderCommandEncoder,
containerSize: containerSize,
texture: input.texture,
textureRotation: self.mainTextureRotation,
rect: rect,
scale: input.scale,
offset: input.offset,
zPosition: 0.0,
device: device
)
}
}
} else {
let (mainVideoState, additionalVideoState, transitionVideoState) = self.transitionState(for: timestamp, mainInput: input.texture, additionalInput: secondInput.first?.texture)
if let transitionVideoState {
self.encodeVideo(
using: renderCommandEncoder,
containerSize: containerSize,
texture: transitionVideoState.texture,
textureRotation: transitionVideoState.textureRotation,
maskTexture: nil,
hasTransparency: false,
position: transitionVideoState.position,
roundness: transitionVideoState.roundness,
alpha: transitionVideoState.alpha,
zPosition: 0.75,
device: device
)
}
self.encodeVideo(
using: renderCommandEncoder,
containerSize: containerSize,
texture: mainVideoState.texture,
textureRotation: mainVideoState.textureRotation,
maskTexture: inputMask,
hasTransparency: hasTransparency,
position: mainVideoState.position,
roundness: mainVideoState.roundness,
alpha: mainVideoState.alpha,
zPosition: 0.0,
device: device
)
if let additionalVideoState {
self.encodeVideo(
using: renderCommandEncoder,
containerSize: containerSize,
texture: additionalVideoState.texture,
textureRotation: additionalVideoState.textureRotation,
maskTexture: nil,
hasTransparency: false,
position: additionalVideoState.position,
roundness: additionalVideoState.roundness,
alpha: additionalVideoState.alpha,
zPosition: 0.5,
device: device
)
}
}
renderCommandEncoder.endEncoding()
return self.cachedTexture!
}
struct GradientColors {
var topColor: simd_float4
var bottomColor: simd_float4
}
func encodeGradient(
using encoder: MTLRenderCommandEncoder,
containerSize: CGSize,
device: MTLDevice
) {
encoder.setRenderPipelineState(self.gradientPipelineState!)
let vertices = verticesDataForRotation(.rotate0Degrees)
let buffer = device.makeBuffer(
bytes: vertices,
length: MemoryLayout<VertexData>.stride * vertices.count,
options: [])
encoder.setVertexBuffer(buffer, offset: 0, index: 0)
encoder.setFragmentBytes(&self.gradientColors, length: MemoryLayout<GradientColors>.size, index: 0)
encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
return nil
}
}
@@ -0,0 +1,128 @@
import Foundation
import AVFoundation
import Metal
import MetalKit
import CoreImage
final class VideoInputPass: DefaultRenderPass {
private var cachedTexture: MTLTexture?
override var fragmentShaderFunctionName: String {
return "bt709ToRGBFragmentShader"
}
override func setup(device: MTLDevice, library: MTLLibrary) {
super.setup(device: device, library: library)
}
func processPixelBuffer(_ pixelBuffer: VideoPixelBuffer, textureCache: CVMetalTextureCache, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
func textureFromPixelBuffer(_ pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, width: Int, height: Int, plane: Int) -> MTLTexture? {
var textureRef : CVMetalTexture?
let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, pixelFormat, width, height, plane, &textureRef)
if status == kCVReturnSuccess, let textureRef {
return CVMetalTextureGetTexture(textureRef)
}
return nil
}
let width = CVPixelBufferGetWidth(pixelBuffer.pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer.pixelBuffer)
guard let inputYTexture = textureFromPixelBuffer(pixelBuffer.pixelBuffer, pixelFormat: .r8Unorm, width: width, height: height, plane: 0),
let inputCbCrTexture = textureFromPixelBuffer(pixelBuffer.pixelBuffer, pixelFormat: .rg8Unorm, width: width >> 1, height: height >> 1, plane: 1) else {
return nil
}
return self.process(yTexture: inputYTexture, cbcrTexture: inputCbCrTexture, width: width, height: height, rotation: pixelBuffer.rotation, device: device, commandBuffer: commandBuffer)
}
func process(yTexture: MTLTexture, cbcrTexture: MTLTexture, width: Int, height: Int, rotation: TextureRotation, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
self.setupVerticesBuffer(device: device, rotation: rotation)
func textureDimensionsForRotation(width: Int, height: Int, rotation: TextureRotation) -> (width: Int, height: Int) {
switch rotation {
case .rotate90Degrees, .rotate270Degrees, .rotate90DegreesMirrored:
return (height, width)
default:
return (width, height)
}
}
let (outputWidth, outputHeight) = textureDimensionsForRotation(width: width, height: height, rotation: rotation)
if self.cachedTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = outputWidth
textureDescriptor.height = outputHeight
textureDescriptor.pixelFormat = self.pixelFormat
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
if let texture = device.makeTexture(descriptor: textureDescriptor) {
self.cachedTexture = texture
}
}
let renderPassDescriptor = MTLRenderPassDescriptor()
renderPassDescriptor.colorAttachments[0].texture = self.cachedTexture!
renderPassDescriptor.colorAttachments[0].loadAction = .dontCare
renderPassDescriptor.colorAttachments[0].storeAction = .store
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.0, green: 0.0, blue: 0.0, alpha: 0.0)
guard let renderCommandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else {
return nil
}
renderCommandEncoder.setViewport(MTLViewport(
originX: 0, originY: 0,
width: Double(outputWidth), height: Double(outputHeight),
znear: -1.0, zfar: 1.0)
)
renderCommandEncoder.setFragmentTexture(yTexture, index: 0)
renderCommandEncoder.setFragmentTexture(cbcrTexture, index: 1)
self.encodeDefaultCommands(using: renderCommandEncoder)
renderCommandEncoder.endEncoding()
return self.cachedTexture
}
}
final class CIInputPass: RenderPass {
private var context: CIContext?
func setup(device: MTLDevice, library: MTLLibrary) {
self.context = CIContext(mtlDevice: device, options: [.workingColorSpace : CGColorSpaceCreateDeviceRGB()])
}
func process(input: MTLTexture, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
return nil
}
private var outputTexture: MTLTexture?
func processCIImage(_ ciImage: CIImage, device: MTLDevice, commandBuffer: MTLCommandBuffer) -> MTLTexture? {
if self.outputTexture == nil {
let textureDescriptor = MTLTextureDescriptor()
textureDescriptor.textureType = .type2D
textureDescriptor.width = Int(ciImage.extent.width)
textureDescriptor.height = Int(ciImage.extent.height)
textureDescriptor.pixelFormat = .bgra8Unorm
textureDescriptor.storageMode = .private
textureDescriptor.usage = [.shaderRead, .shaderWrite, .renderTarget]
guard let texture = device.makeTexture(descriptor: textureDescriptor) else {
return nil
}
self.outputTexture = texture
texture.label = "outlineOutputTexture"
}
guard let outputTexture = self.outputTexture, let context = self.context else {
return nil
}
let transformedImage = ciImage.transformed(by: CGAffineTransformMakeScale(1.0, -1.0).translatedBy(x: 0.0, y: -ciImage.extent.height))
let renderDestination = CIRenderDestination(mtlTexture: outputTexture, commandBuffer: commandBuffer)
_ = try? context.startTask(toRender: transformedImage, to: renderDestination)
return outputTexture
}
}