Merge commit '7621e2f8dec938cf48181c8b10afc9b01f444e68' into beta

This commit is contained in:
Ilya Laktyushin
2025-12-06 02:17:48 +04:00
commit 8344b97e03
28070 changed files with 7995182 additions and 0 deletions
+69
View File
@@ -0,0 +1,69 @@
load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
load(
"@build_bazel_rules_apple//apple:resources.bzl",
"apple_resource_bundle",
"apple_resource_group",
)
load("//build-system/bazel-utils:plist_fragment.bzl",
"plist_fragment",
)
filegroup(
name = "CameraMetalResources",
srcs = glob([
"MetalResources/**/*.*",
]),
visibility = ["//visibility:public"],
)
plist_fragment(
name = "CameraBundleInfoPlist",
extension = "plist",
template =
"""
<key>CFBundleIdentifier</key>
<string>org.telegram.Camera</string>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleName</key>
<string>Camera</string>
"""
)
apple_resource_bundle(
name = "CameraBundle",
infoplists = [
":CameraBundleInfoPlist",
],
resources = [
":CameraMetalResources",
],
)
swift_library(
name = "Camera",
module_name = "Camera",
srcs = glob([
"Sources/**/*.swift",
]),
copts = [
"-warnings-as-errors",
],
data = [
":CameraBundle",
],
deps = [
"//submodules/SSignalKit/SwiftSignalKit:SwiftSignalKit",
"//submodules/AsyncDisplayKit:AsyncDisplayKit",
"//submodules/Display:Display",
"//submodules/ImageBlur:ImageBlur",
"//submodules/TelegramCore:TelegramCore",
"//submodules/Utils/DeviceModel",
"//submodules/rlottie:RLottieBinding",
"//submodules/AppBundle",
"//submodules/GZip",
],
visibility = [
"//visibility:public",
],
)
+22
View File
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>FMWK</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>$(CURRENT_PROJECT_VERSION)</string>
</dict>
</plist>
@@ -0,0 +1,30 @@
#include <metal_stdlib>
using namespace metal;
// Vertex input/output structure for passing results from vertex shader to fragment shader
struct VertexIO
{
float4 position [[position]];
float2 textureCoord [[user(texturecoord)]];
};
// Vertex shader for a textured quad
vertex VertexIO vertexPassThrough(const device packed_float4 *pPosition [[ buffer(0) ]],
const device packed_float2 *pTexCoords [[ buffer(1) ]],
uint vid [[ vertex_id ]])
{
VertexIO outVertex;
outVertex.position = pPosition[vid];
outVertex.textureCoord = pTexCoords[vid];
return outVertex;
}
// Fragment shader for a textured quad
fragment half4 fragmentPassThrough(VertexIO inputFragment [[ stage_in ]],
texture2d<half> inputTexture [[ texture(0) ]],
sampler samplr [[ sampler(0) ]])
{
return inputTexture.sample(samplr, inputFragment.textureCoord);
}
+11
View File
@@ -0,0 +1,11 @@
#import <UIKit/UIKit.h>
//! Project version number for Camera.
FOUNDATION_EXPORT double CameraVersionNumber;
//! Project version string for Camera.
FOUNDATION_EXPORT const unsigned char CameraVersionString[];
// In this header, you should import all the public headers of your framework using statements like #import <Camera/PublicHeader.h>
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,338 @@
import Foundation
import AVFoundation
import SwiftSignalKit
import TelegramCore
private let defaultFPS: Double = 30.0
final class CameraDevice {
var position: Camera.Position = .back
deinit {
if let videoDevice = self.videoDevice {
self.unsubscribeFromChanges(videoDevice)
}
}
public private(set) var videoDevice: AVCaptureDevice? = nil {
didSet {
if let previousVideoDevice = oldValue {
self.unsubscribeFromChanges(previousVideoDevice)
}
self.videoDevicePromise.set(.single(self.videoDevice))
if let videoDevice = self.videoDevice {
self.subscribeForChanges(videoDevice)
}
}
}
private var videoDevicePromise = Promise<AVCaptureDevice?>()
public private(set) var audioDevice: AVCaptureDevice? = nil
func configure(for session: CameraSession, position: Camera.Position, dual: Bool, switchAudio: Bool) {
self.position = position
var selectedDevice: AVCaptureDevice?
if #available(iOS 13.0, *), position != .front && !dual {
if let device = AVCaptureDevice.default(.builtInTripleCamera, for: .video, position: position) {
selectedDevice = device
} else if let device = AVCaptureDevice.default(.builtInDualCamera, for: .video, position: position) {
selectedDevice = device
} else if let device = AVCaptureDevice.default(.builtInDualWideCamera, for: .video, position: position) {
selectedDevice = device
} else if let device = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera, .builtInTelephotoCamera], mediaType: .video, position: position).devices.first {
selectedDevice = device
}
} else {
if selectedDevice == nil {
selectedDevice = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera, .builtInTelephotoCamera], mediaType: .video, position: position).devices.first
}
}
if selectedDevice == nil, #available(iOS 13.0, *) {
let allDevices = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInDualCamera, .builtInTripleCamera, .builtInTelephotoCamera, .builtInDualWideCamera, .builtInTrueDepthCamera, .builtInWideAngleCamera, .builtInUltraWideCamera], mediaType: .video, position: position).devices
Logger.shared.log("Camera", "No device selected, availabled devices: \(allDevices)")
}
self.videoDevice = selectedDevice
self.videoDevicePromise.set(.single(selectedDevice))
if switchAudio {
self.audioDevice = AVCaptureDevice.default(for: .audio)
}
}
func configureDeviceFormat(maxDimensions: CMVideoDimensions, maxFramerate: Double) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
var maxWidth: Int32 = 0
var maxHeight: Int32 = 0
var hasSecondaryZoomLevels = false
var candidates: [AVCaptureDevice.Format] = []
var photoCandidates: [AVCaptureDevice.Format] = []
outer: for format in device.formats {
if format.mediaType != .video || format.value(forKey: "isPhotoFormat") as? Bool == true {
continue
}
let dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription)
if dimensions.width >= maxWidth && dimensions.width <= maxDimensions.width && dimensions.height >= maxHeight && dimensions.height <= maxDimensions.height {
if dimensions.width > maxWidth {
hasSecondaryZoomLevels = false
candidates.removeAll()
}
let subtype = CMFormatDescriptionGetMediaSubType(format.formatDescription)
if subtype == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange {
for range in format.videoSupportedFrameRateRanges {
if range.maxFrameRate > 60 {
continue outer
}
}
maxWidth = dimensions.width
maxHeight = dimensions.height
if #available(iOS 16.0, *), !format.secondaryNativeResolutionZoomFactors.isEmpty {
hasSecondaryZoomLevels = true
candidates.append(format)
if format.isHighPhotoQualitySupported {
photoCandidates.append(format)
}
} else if !hasSecondaryZoomLevels {
candidates.append(format)
if #available(iOS 15.0, *), format.isHighPhotoQualitySupported {
photoCandidates.append(format)
}
}
}
}
}
if !candidates.isEmpty {
var bestFormat: AVCaptureDevice.Format?
photoOuter: for format in photoCandidates {
for range in format.videoSupportedFrameRateRanges {
if range.maxFrameRate > maxFramerate {
continue photoOuter
}
bestFormat = format
}
}
if bestFormat == nil {
outer: for format in candidates {
for range in format.videoSupportedFrameRateRanges {
if range.maxFrameRate > maxFramerate {
continue outer
}
bestFormat = format
}
}
}
if bestFormat == nil {
bestFormat = candidates.last
}
device.activeFormat = bestFormat!
Logger.shared.log("Camera", "Selected format:")
Logger.shared.log("Camera", bestFormat!.description)
} else {
Logger.shared.log("Camera", "No format selected")
}
Logger.shared.log("Camera", "Available formats:")
for format in device.formats {
Logger.shared.log("Camera", format.description)
}
if let targetFPS = device.actualFPS(maxFramerate) {
device.activeVideoMinFrameDuration = targetFPS.duration
device.activeVideoMaxFrameDuration = targetFPS.duration
}
if device.isLowLightBoostSupported {
device.automaticallyEnablesLowLightBoostWhenAvailable = true
}
if device.isExposureModeSupported(.continuousAutoExposure) {
device.exposureMode = .continuousAutoExposure
}
}
}
func transaction(_ device: AVCaptureDevice, update: (AVCaptureDevice) -> Void) {
if let _ = try? device.lockForConfiguration() {
update(device)
device.unlockForConfiguration()
}
}
private func subscribeForChanges(_ device: AVCaptureDevice) {
NotificationCenter.default.addObserver(self, selector: #selector(self.subjectAreaChanged), name: Notification.Name.AVCaptureDeviceSubjectAreaDidChange, object: device)
}
private func unsubscribeFromChanges(_ device: AVCaptureDevice) {
NotificationCenter.default.removeObserver(self, name: Notification.Name.AVCaptureDeviceSubjectAreaDidChange, object: device)
}
@objc private func subjectAreaChanged() {
self.setFocusPoint(CGPoint(x: 0.5, y: 0.5), focusMode: .continuousAutoFocus, exposureMode: .continuousAutoExposure, monitorSubjectAreaChange: false)
}
var fps: Double = defaultFPS {
didSet {
guard let device = self.videoDevice, let targetFPS = device.actualFPS(Double(self.fps)) else {
return
}
self.fps = targetFPS.fps
self.transaction(device) { device in
device.activeVideoMinFrameDuration = targetFPS.duration
device.activeVideoMaxFrameDuration = targetFPS.duration
}
}
}
var isTorchAvailable: Signal<Bool, NoError> {
return self.videoDevicePromise.get()
|> mapToSignal { device -> Signal<Bool, NoError> in
return Signal { subscriber in
guard let device else {
return EmptyDisposable
}
subscriber.putNext(device.isFlashAvailable)
let observer = device.observe(\.isFlashAvailable, options: [.new], changeHandler: { device, _ in
subscriber.putNext(device.isFlashAvailable)
})
return ActionDisposable {
observer.invalidate()
}
}
|> distinctUntilChanged
}
}
var isAdjustingFocus: Signal<Bool, NoError> {
return self.videoDevicePromise.get()
|> mapToSignal { device -> Signal<Bool, NoError> in
return Signal { subscriber in
guard let device else {
return EmptyDisposable
}
subscriber.putNext(device.isAdjustingFocus)
let observer = device.observe(\.isAdjustingFocus, options: [.new], changeHandler: { device, _ in
subscriber.putNext(device.isAdjustingFocus)
})
return ActionDisposable {
observer.invalidate()
}
}
|> distinctUntilChanged
}
}
func setFocusPoint(_ point: CGPoint, focusMode: Camera.FocusMode, exposureMode: Camera.ExposureMode, monitorSubjectAreaChange: Bool) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
if device.isExposurePointOfInterestSupported && device.isExposureModeSupported(exposureMode) {
device.exposurePointOfInterest = point
device.exposureMode = exposureMode
}
if device.isFocusPointOfInterestSupported && device.isFocusModeSupported(focusMode) {
device.focusPointOfInterest = point
device.focusMode = focusMode
}
device.isSubjectAreaChangeMonitoringEnabled = monitorSubjectAreaChange
if abs(device.exposureTargetBias) > 0.0 {
device.setExposureTargetBias(0.0)
}
}
}
func setExposureTargetBias(_ bias: Float) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
let extremum = (bias >= 0) ? device.maxExposureTargetBias : device.minExposureTargetBias;
let value = abs(bias) * extremum * 0.85
device.setExposureTargetBias(value, completionHandler: nil)
}
}
func setTorchActive(_ active: Bool) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
let torchMode: AVCaptureDevice.TorchMode = active ? .on : .off
if device.isTorchModeSupported(torchMode) {
device.torchMode = active ? .on : .off
}
}
}
func setTorchMode(_ flashMode: AVCaptureDevice.FlashMode) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
let torchMode: AVCaptureDevice.TorchMode
switch flashMode {
case .on:
torchMode = .on
case .off:
torchMode = .off
case .auto:
torchMode = .auto
@unknown default:
torchMode = .off
}
if device.isTorchModeSupported(torchMode) {
device.torchMode = torchMode
}
}
}
func setZoomLevel(_ zoomLevel: CGFloat) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
device.videoZoomFactor = max(device.neutralZoomFactor, min(10.0, device.neutralZoomFactor + zoomLevel))
}
}
func setZoomDelta(_ zoomDelta: CGFloat) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
device.videoZoomFactor = max(1.0, min(10.0, device.videoZoomFactor * zoomDelta))
}
}
func rampZoom(_ zoomLevel: CGFloat, rate: CGFloat) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
device.ramp(toVideoZoomFactor: zoomLevel, withRate: Float(rate))
}
}
func resetZoom(neutral: Bool = true) {
guard let device = self.videoDevice else {
return
}
self.transaction(device) { device in
device.videoZoomFactor = neutral ? device.neutralZoomFactor : device.minAvailableVideoZoomFactor
}
}
}
@@ -0,0 +1,59 @@
import AVFoundation
import TelegramCore
class CameraInput {
var videoInput: AVCaptureDeviceInput?
private var audioInput: AVCaptureDeviceInput?
func configure(for session: CameraSession, device: CameraDevice, audio: Bool) {
if let videoDevice = device.videoDevice {
self.configureVideoInput(for: session, device: videoDevice)
}
if audio, let audioDevice = device.audioDevice {
self.configureAudioInput(for: session, device: audioDevice)
}
}
func invalidate(for session: CameraSession, switchAudio: Bool = true) {
for input in session.session.inputs {
if !switchAudio && input === self.audioInput {
continue
}
session.session.removeInput(input)
}
}
private func configureVideoInput(for session: CameraSession, device: AVCaptureDevice) {
if let currentVideoInput = self.videoInput {
session.session.removeInput(currentVideoInput)
self.videoInput = nil
}
if let videoInput = try? AVCaptureDeviceInput(device: device) {
self.videoInput = videoInput
if session.session.canAddInput(videoInput) {
if session.hasMultiCam {
session.session.addInputWithNoConnections(videoInput)
} else {
session.session.addInput(videoInput)
}
} else {
Logger.shared.log("Camera", "Can't add video input")
}
}
}
private func configureAudioInput(for session: CameraSession, device: AVCaptureDevice) {
if let currentAudioInput = self.audioInput {
session.session.removeInput(currentAudioInput)
self.audioInput = nil
}
if let audioInput = try? AVCaptureDeviceInput(device: device) {
self.audioInput = audioInput
if session.session.canAddInput(audioInput) {
session.session.addInput(audioInput)
} else {
Logger.shared.log("Camera", "Can't add audio input")
}
}
}
}
@@ -0,0 +1,76 @@
import Foundation
import DeviceModel
public extension Camera {
enum Metrics {
case singleCamera
case iPhone14
case iPhone14Plus
case iPhone14Pro
case iPhone14ProMax
case iPhone15
case iPhone15Plus
case iPhone15Pro
case iPhone15ProMax
case iPhone17
case iPhone17Pro
case iPhoneAir
case unknown
public init(model: DeviceModel) {
switch model {
case .iPodTouch1, .iPodTouch2, .iPodTouch3, .iPodTouch4, .iPodTouch5, .iPodTouch6, .iPodTouch7:
self = .singleCamera
case .iPhone14:
self = .iPhone14
case .iPhone14Plus:
self = .iPhone14Plus
case .iPhone14Pro:
self = .iPhone14Pro
case .iPhone14ProMax:
self = .iPhone14ProMax
case .iPhone15:
self = .iPhone15
case .iPhone15Plus:
self = .iPhone15Plus
case .iPhone15Pro:
self = .iPhone15Pro
case .iPhone15ProMax:
self = .iPhone15ProMax
case .iPhone16Pro:
self = .iPhone15Pro
case .iPhone16ProMax:
self = .iPhone15ProMax
case .iPhone17:
self = .iPhone17
case .iPhone17Pro, .iPhone17ProMax:
self = .iPhone17Pro
case .iPhoneAir:
self = .iPhoneAir
case .unknown:
self = .unknown
default:
self = .unknown
}
}
public var zoomLevels: [Float] {
switch self {
case .singleCamera:
return [1.0]
case .iPhone14, .iPhone14Plus, .iPhone15, .iPhone15Plus, .iPhone17:
return [0.5, 1.0, 2.0]
case .iPhone14Pro, .iPhone14ProMax, .iPhone15Pro:
return [0.5, 1.0, 2.0, 3.0]
case .iPhone15ProMax:
return [0.5, 1.0, 2.0, 5.0]
case .iPhone17Pro:
return [0.5, 1.0, 2.0, 8.0]
case .iPhoneAir:
return [1.0, 2.0]
case .unknown:
return [1.0, 2.0]
}
}
}
}
@@ -0,0 +1,690 @@
import Foundation
import AVFoundation
import UIKit
import Display
import SwiftSignalKit
import CoreImage
import Vision
import VideoToolbox
import TelegramCore
public enum VideoCaptureResult: Equatable {
public struct Result {
public let path: String
public let thumbnail: UIImage
public let isMirrored: Bool
public let dimensions: CGSize
}
case finished(main: Result, additional: Result?, duration: Double, positionChangeTimestamps: [(Bool, Double)], captureTimestamp: Double)
case failed
public static func == (lhs: VideoCaptureResult, rhs: VideoCaptureResult) -> Bool {
switch lhs {
case .failed:
if case .failed = rhs {
return true
} else {
return false
}
case let .finished(_, _, lhsDuration, lhsChangeTimestamps, lhsTimestamp):
if case let .finished(_, _, rhsDuration, rhsChangeTimestamps, rhsTimestamp) = rhs, lhsDuration == rhsDuration, lhsTimestamp == rhsTimestamp {
if lhsChangeTimestamps.count != rhsChangeTimestamps.count {
return false
}
return true
} else {
return false
}
}
}
}
public struct CameraCode: Equatable {
public enum CodeType {
case qr
}
public let type: CodeType
public let message: String
public let corners: [CGPoint]
public init(type: CameraCode.CodeType, message: String, corners: [CGPoint]) {
self.type = type
self.message = message
self.corners = corners
}
public var boundingBox: CGRect {
let x = self.corners.map { $0.x }
let y = self.corners.map { $0.y }
if let minX = x.min(), let minY = y.min(), let maxX = x.max(), let maxY = y.max() {
return CGRect(x: minX, y: minY, width: abs(maxX - minX), height: abs(maxY - minY))
}
return CGRect.null
}
public var rotation: CGFloat {
guard self.corners.count == 4 else {
return 0.0
}
let topLeft = self.corners[1]
let topRight = self.corners[2]
let dx = topRight.x - topLeft.x
let dy = topRight.y - topLeft.y
return atan2(dy, dx) - .pi / 2.0
}
public static func == (lhs: CameraCode, rhs: CameraCode) -> Bool {
if lhs.type != rhs.type {
return false
}
if lhs.message != rhs.message {
return false
}
if lhs.corners != rhs.corners {
return false
}
return true
}
}
final class CameraOutput: NSObject {
let exclusive: Bool
let ciContext: CIContext
let colorSpace: CGColorSpace
let isVideoMessage: Bool
var hasAudio: Bool = false
let photoOutput = AVCapturePhotoOutput()
let videoOutput = AVCaptureVideoDataOutput()
let audioOutput = AVCaptureAudioDataOutput()
let metadataOutput = AVCaptureMetadataOutput()
private var photoConnection: AVCaptureConnection?
private var videoConnection: AVCaptureConnection?
private var previewConnection: AVCaptureConnection?
private var roundVideoFilter: CameraRoundLegacyVideoFilter?
private let semaphore = DispatchSemaphore(value: 1)
private let videoQueue = DispatchQueue(label: "", qos: .userInitiated)
private let audioQueue = DispatchQueue(label: "")
private let metadataQueue = DispatchQueue(label: "")
private var photoCaptureRequests: [Int64: PhotoCaptureContext] = [:]
private var videoRecorder: VideoRecorder?
private var captureOrientation: AVCaptureVideoOrientation = .portrait
var processSampleBuffer: ((CMSampleBuffer, CVImageBuffer, AVCaptureConnection) -> Void)?
var processAudioBuffer: ((CMSampleBuffer) -> Void)?
var processCodes: (([CameraCode]) -> Void)?
init(exclusive: Bool, ciContext: CIContext, colorSpace: CGColorSpace, use32BGRA: Bool = false) {
self.exclusive = exclusive
self.ciContext = ciContext
self.colorSpace = colorSpace
self.isVideoMessage = use32BGRA
super.init()
if #available(iOS 13.0, *) {
self.photoOutput.maxPhotoQualityPrioritization = .balanced
}
self.videoOutput.alwaysDiscardsLateVideoFrames = false
self.videoOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey: use32BGRA ? kCVPixelFormatType_32BGRA : kCVPixelFormatType_420YpCbCr8BiPlanarFullRange] as [String : Any]
}
deinit {
self.videoOutput.setSampleBufferDelegate(nil, queue: nil)
self.audioOutput.setSampleBufferDelegate(nil, queue: nil)
}
func configure(for session: CameraSession, device: CameraDevice, input: CameraInput, previewView: CameraSimplePreviewView?, audio: Bool, photo: Bool, metadata: Bool) {
if session.session.canAddOutput(self.videoOutput) {
if session.hasMultiCam {
session.session.addOutputWithNoConnections(self.videoOutput)
} else {
session.session.addOutput(self.videoOutput)
}
self.videoOutput.setSampleBufferDelegate(self, queue: self.videoQueue)
} else {
Logger.shared.log("Camera", "Can't add video output")
}
if audio {
self.hasAudio = true
if session.session.canAddOutput(self.audioOutput) {
session.session.addOutput(self.audioOutput)
self.audioOutput.setSampleBufferDelegate(self, queue: self.audioQueue)
} else {
Logger.shared.log("Camera", "Can't add audio output")
}
}
if photo, session.session.canAddOutput(self.photoOutput) {
if session.hasMultiCam {
session.session.addOutputWithNoConnections(self.photoOutput)
} else {
session.session.addOutput(self.photoOutput)
}
} else {
Logger.shared.log("Camera", "Can't add photo output")
}
if metadata, session.session.canAddOutput(self.metadataOutput) {
session.session.addOutput(self.metadataOutput)
self.metadataOutput.setMetadataObjectsDelegate(self, queue: self.metadataQueue)
if self.metadataOutput.availableMetadataObjectTypes.contains(.qr) {
self.metadataOutput.metadataObjectTypes = [.qr]
}
}
if #available(iOS 13.0, *), session.hasMultiCam {
if let device = device.videoDevice, let ports = input.videoInput?.ports(for: AVMediaType.video, sourceDeviceType: device.deviceType, sourceDevicePosition: device.position) {
if let previewView {
let previewConnection = AVCaptureConnection(inputPort: ports.first!, videoPreviewLayer: previewView.videoPreviewLayer)
if session.session.canAddConnection(previewConnection) {
session.session.addConnection(previewConnection)
self.previewConnection = previewConnection
} else {
Logger.shared.log("Camera", "Can't add preview connection")
}
}
let videoConnection = AVCaptureConnection(inputPorts: ports, output: self.videoOutput)
if session.session.canAddConnection(videoConnection) {
session.session.addConnection(videoConnection)
self.videoConnection = videoConnection
} else {
Logger.shared.log("Camera", "Can't add video connection")
}
if photo {
let photoConnection = AVCaptureConnection(inputPorts: ports, output: self.photoOutput)
if session.session.canAddConnection(photoConnection) {
session.session.addConnection(photoConnection)
self.photoConnection = photoConnection
}
}
} else {
Logger.shared.log("Camera", "Can't get video port")
}
}
}
func invalidate(for session: CameraSession, switchAudio: Bool = true) {
if #available(iOS 13.0, *) {
if let previewConnection = self.previewConnection {
if session.session.connections.contains(where: { $0 === previewConnection }) {
session.session.removeConnection(previewConnection)
}
self.previewConnection = nil
}
if let videoConnection = self.videoConnection {
if session.session.connections.contains(where: { $0 === videoConnection }) {
session.session.removeConnection(videoConnection)
}
self.videoConnection = nil
}
if let photoConnection = self.photoConnection {
if session.session.connections.contains(where: { $0 === photoConnection }) {
session.session.removeConnection(photoConnection)
}
self.photoConnection = nil
}
}
if session.session.outputs.contains(where: { $0 === self.videoOutput }) {
session.session.removeOutput(self.videoOutput)
}
if switchAudio, session.session.outputs.contains(where: { $0 === self.audioOutput }) {
session.session.removeOutput(self.audioOutput)
}
if session.session.outputs.contains(where: { $0 === self.photoOutput }) {
session.session.removeOutput(self.photoOutput)
}
if session.session.outputs.contains(where: { $0 === self.metadataOutput }) {
session.session.removeOutput(self.metadataOutput)
}
}
func configureVideoStabilization() {
if let videoDataOutputConnection = self.videoOutput.connection(with: .video) {
if videoDataOutputConnection.isVideoStabilizationSupported {
videoDataOutputConnection.preferredVideoStabilizationMode = .standard
// videoDataOutputConnection.preferredVideoStabilizationMode = self.isVideoMessage ? .cinematic : .standard
}
}
}
var isFlashActive: Signal<Bool, NoError> {
return Signal { [weak self] subscriber in
guard let self else {
return EmptyDisposable
}
subscriber.putNext(self.photoOutput.isFlashScene)
let observer = self.photoOutput.observe(\.isFlashScene, options: [.new], changeHandler: { device, _ in
subscriber.putNext(self.photoOutput.isFlashScene)
})
return ActionDisposable {
observer.invalidate()
}
}
|> distinctUntilChanged
}
func takePhoto(orientation: AVCaptureVideoOrientation, flashMode: AVCaptureDevice.FlashMode) -> Signal<PhotoCaptureResult, NoError> {
var mirror = false
if let connection = self.photoOutput.connection(with: .video) {
connection.videoOrientation = orientation
if #available(iOS 13.0, *) {
mirror = connection.inputPorts.first?.sourceDevicePosition == .front
}
}
let settings = AVCapturePhotoSettings(format: [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)])
settings.flashMode = mirror ? .off : flashMode
if let previewPhotoPixelFormatType = settings.availablePreviewPhotoPixelFormatTypes.first {
settings.previewPhotoFormat = [kCVPixelBufferPixelFormatTypeKey as String: previewPhotoPixelFormatType]
}
if #available(iOS 13.0, *) {
if self.photoOutput.maxPhotoQualityPrioritization != .speed {
settings.photoQualityPrioritization = .balanced
} else {
settings.photoQualityPrioritization = .speed
}
}
#if targetEnvironment(simulator)
let image = generateImage(CGSize(width: 1080, height: 1920), opaque: true, scale: 1.0, rotatedContext: { size, context in
let colors: [UIColor] = [UIColor(rgb: 0xff00ff), UIColor(rgb: 0xff0000), UIColor(rgb: 0x00ffff), UIColor(rgb: 0x00ff00)]
if let randomColor = colors.randomElement() {
context.setFillColor(randomColor.cgColor)
}
context.fill(CGRect(origin: .zero, size: size))
})!
return .single(.began)
|> then(
.single(.finished(image, nil, CACurrentMediaTime())) |> delay(0.5, queue: Queue.concurrentDefaultQueue())
)
#else
let uniqueId = settings.uniqueID
let photoCapture = PhotoCaptureContext(ciContext: self.ciContext, settings: settings, orientation: orientation, mirror: mirror)
self.photoCaptureRequests[uniqueId] = photoCapture
self.photoOutput.capturePhoto(with: settings, delegate: photoCapture)
return photoCapture.signal
|> afterDisposed { [weak self] in
self?.photoCaptureRequests.removeValue(forKey: uniqueId)
}
#endif
}
var isRecording: Bool {
return self.videoRecorder != nil
}
enum RecorderMode {
case `default`
case roundVideo
case dualCamera
}
private var currentMode: RecorderMode = .default
private var recordingCompletionPipe = ValuePipe<VideoCaptureResult>()
func startRecording(mode: RecorderMode, position: Camera.Position? = nil, orientation: AVCaptureVideoOrientation, additionalOutput: CameraOutput? = nil) -> Signal<CameraRecordingData, CameraRecordingError> {
guard self.videoRecorder == nil else {
return .complete()
}
Logger.shared.log("CameraOutput", "startRecording")
self.currentMode = mode
self.lastSampleTimestamp = nil
self.captureOrientation = orientation
var orientation = orientation
let dimensions: CGSize
let videoSettings: [String: Any]
if case .roundVideo = mode {
dimensions = videoMessageDimensions.cgSize
orientation = .landscapeRight
let compressionProperties: [String: Any] = [
AVVideoAverageBitRateKey: 1000 * 1000,
AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel,
AVVideoH264EntropyModeKey: AVVideoH264EntropyModeCABAC
]
videoSettings = [
AVVideoCodecKey: AVVideoCodecType.h264,
AVVideoCompressionPropertiesKey: compressionProperties,
AVVideoWidthKey: Int(dimensions.width),
AVVideoHeightKey: Int(dimensions.height)
]
} else {
let codecType: AVVideoCodecType = hasHEVCHardwareEncoder ? .hevc : .h264
if orientation == .landscapeLeft || orientation == .landscapeRight {
dimensions = CGSize(width: 1920, height: 1080)
} else {
dimensions = CGSize(width: 1080, height: 1920)
}
guard let settings = self.videoOutput.recommendedVideoSettings(forVideoCodecType: codecType, assetWriterOutputFileType: .mp4) else {
return .complete()
}
videoSettings = settings
}
let audioSettings = self.audioOutput.recommendedAudioSettingsForAssetWriter(writingTo: .mp4) ?? [:]
if self.hasAudio && audioSettings.isEmpty {
Logger.shared.log("Camera", "Audio settings are empty on recording start")
return .fail(.audioInitializationError)
}
let outputFileName = NSUUID().uuidString
let outputFilePath = NSTemporaryDirectory() + outputFileName + ".mp4"
let outputFileURL = URL(fileURLWithPath: outputFilePath)
let videoRecorder = VideoRecorder(
configuration: VideoRecorder.Configuration(videoSettings: videoSettings, audioSettings: audioSettings),
ciContext: self.ciContext,
orientation: orientation,
fileUrl: outputFileURL,
completion: { [weak self] result in
guard let self else {
return
}
if case let .success(transitionImage, duration, positionChangeTimestamps) = result {
self.recordingCompletionPipe.putNext(
.finished(
main: VideoCaptureResult.Result(
path: outputFilePath,
thumbnail: transitionImage ?? UIImage(),
isMirrored: false,
dimensions: dimensions
),
additional: nil,
duration: duration,
positionChangeTimestamps: positionChangeTimestamps.map { ($0 == .front, $1) },
captureTimestamp: CACurrentMediaTime()
)
)
} else {
self.recordingCompletionPipe.putNext(.failed)
}
}
)
videoRecorder?.start()
self.videoRecorder = videoRecorder
if case .dualCamera = mode, let position {
videoRecorder?.markPositionChange(position: position, time: .zero)
} else if case .roundVideo = mode {
additionalOutput?.masterOutput = self
}
return Signal { subscriber in
let timer = SwiftSignalKit.Timer(timeout: 0.033, repeat: true, completion: { [weak videoRecorder] in
let recordingData = CameraRecordingData(duration: videoRecorder?.duration ?? 0.0, filePath: outputFilePath)
subscriber.putNext(recordingData)
}, queue: Queue.mainQueue())
timer.start()
return ActionDisposable {
timer.invalidate()
}
}
}
func stopRecording() -> Signal<VideoCaptureResult, NoError> {
guard let videoRecorder = self.videoRecorder, videoRecorder.isRecording else {
return .complete()
}
videoRecorder.stop()
return self.recordingCompletionPipe.signal()
|> take(1)
|> afterDisposed {
self.videoRecorder = nil
}
}
var transitionImage: UIImage? {
return self.videoRecorder?.transitionImage
}
private weak var masterOutput: CameraOutput?
private var lastSampleTimestamp: CMTime?
private var needsCrossfadeTransition = false
private var crossfadeTransitionStart: Double = 0.0
private var needsSwitchSampleOffset = false
private var lastAudioSampleTime: CMTime?
private var videoSwitchSampleTimeOffset: CMTime?
func processVideoRecording(_ sampleBuffer: CMSampleBuffer, fromAdditionalOutput: Bool) {
guard let videoRecorder = self.videoRecorder, videoRecorder.isRecording else {
return
}
guard let formatDescriptor = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return
}
let type = CMFormatDescriptionGetMediaType(formatDescriptor)
if case .roundVideo = self.currentMode, type == kCMMediaType_Video {
let currentTimestamp = CACurrentMediaTime()
let duration: Double = 0.2
if !self.exclusive {
var transitionFactor: CGFloat = 0.0
if case .front = self.currentPosition {
transitionFactor = 1.0
if self.lastSwitchTimestamp > 0.0, currentTimestamp - self.lastSwitchTimestamp < duration {
transitionFactor = max(0.0, (currentTimestamp - self.lastSwitchTimestamp) / duration)
}
} else {
transitionFactor = 0.0
if self.lastSwitchTimestamp > 0.0, currentTimestamp - self.lastSwitchTimestamp < duration {
transitionFactor = 1.0 - max(0.0, (currentTimestamp - self.lastSwitchTimestamp) / duration)
}
}
if (transitionFactor == 1.0 && fromAdditionalOutput)
|| (transitionFactor == 0.0 && !fromAdditionalOutput)
|| (transitionFactor > 0.0 && transitionFactor < 1.0) {
if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: fromAdditionalOutput, transitionFactor: transitionFactor) {
let presentationTime = CMSampleBufferGetPresentationTimeStamp(processedSampleBuffer)
if let lastSampleTimestamp = self.lastSampleTimestamp, lastSampleTimestamp > presentationTime {
} else {
videoRecorder.appendSampleBuffer(processedSampleBuffer)
self.lastSampleTimestamp = presentationTime
}
}
}
} else {
var additional = self.currentPosition == .front
var transitionFactor = self.currentPosition == .front ? 1.0 : 0.0
if self.lastSwitchTimestamp > 0.0 {
if self.needsCrossfadeTransition {
self.needsCrossfadeTransition = false
self.crossfadeTransitionStart = currentTimestamp + 0.03
self.needsSwitchSampleOffset = true
}
if self.crossfadeTransitionStart > 0.0, currentTimestamp - self.crossfadeTransitionStart < duration {
if case .front = self.currentPosition {
transitionFactor = max(0.0, (currentTimestamp - self.crossfadeTransitionStart) / duration)
} else {
transitionFactor = 1.0 - max(0.0, (currentTimestamp - self.crossfadeTransitionStart) / duration)
}
} else if currentTimestamp - self.lastSwitchTimestamp < 0.05 {
additional = !additional
transitionFactor = 1.0 - transitionFactor
self.needsCrossfadeTransition = true
}
}
if let processedSampleBuffer = self.processRoundVideoSampleBuffer(sampleBuffer, additional: additional, transitionFactor: transitionFactor) {
videoRecorder.appendSampleBuffer(processedSampleBuffer)
} else {
videoRecorder.appendSampleBuffer(sampleBuffer)
}
}
} else {
if type == kCMMediaType_Audio {
if self.needsSwitchSampleOffset {
self.needsSwitchSampleOffset = false
if let lastAudioSampleTime = self.lastAudioSampleTime {
let videoSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
let offset = videoSampleTime - lastAudioSampleTime
if let current = self.videoSwitchSampleTimeOffset {
self.videoSwitchSampleTimeOffset = current + offset
} else {
self.videoSwitchSampleTimeOffset = offset
}
self.lastAudioSampleTime = nil
}
}
self.lastAudioSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + CMSampleBufferGetDuration(sampleBuffer)
}
videoRecorder.appendSampleBuffer(sampleBuffer)
}
}
private func processRoundVideoSampleBuffer(_ sampleBuffer: CMSampleBuffer, additional: Bool, transitionFactor: CGFloat) -> CMSampleBuffer? {
guard let videoPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer), let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return nil
}
self.semaphore.wait()
let mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription)
let extensions = CMFormatDescriptionGetExtensions(formatDescription) as! [String: Any]
var updatedExtensions = extensions
updatedExtensions["CVBytesPerRow"] = videoMessageDimensions.width * 4
var newFormatDescription: CMFormatDescription?
var status = CMVideoFormatDescriptionCreate(allocator: nil, codecType: mediaSubType, width: videoMessageDimensions.width, height: videoMessageDimensions.height, extensions: updatedExtensions as CFDictionary, formatDescriptionOut: &newFormatDescription)
guard status == noErr, let newFormatDescription else {
self.semaphore.signal()
return nil
}
let filter: CameraRoundLegacyVideoFilter
if let current = self.roundVideoFilter {
filter = current
} else {
filter = CameraRoundLegacyVideoFilter(ciContext: self.ciContext, colorSpace: self.colorSpace, simple: self.exclusive)
self.roundVideoFilter = filter
}
if !filter.isPrepared {
filter.prepare(with: newFormatDescription, outputRetainedBufferCountHint: 4)
}
guard let newPixelBuffer = filter.render(pixelBuffer: videoPixelBuffer, additional: additional, captureOrientation: self.captureOrientation, transitionFactor: transitionFactor) else {
self.semaphore.signal()
return nil
}
var sampleTimingInfo: CMSampleTimingInfo = .invalid
CMSampleBufferGetSampleTimingInfo(sampleBuffer, at: 0, timingInfoOut: &sampleTimingInfo)
if let videoSwitchSampleTimeOffset = self.videoSwitchSampleTimeOffset {
sampleTimingInfo.decodeTimeStamp = sampleTimingInfo.decodeTimeStamp - videoSwitchSampleTimeOffset
sampleTimingInfo.presentationTimeStamp = sampleTimingInfo.presentationTimeStamp - videoSwitchSampleTimeOffset
}
var newSampleBuffer: CMSampleBuffer?
status = CMSampleBufferCreateForImageBuffer(
allocator: kCFAllocatorDefault,
imageBuffer: newPixelBuffer,
dataReady: true,
makeDataReadyCallback: nil,
refcon: nil,
formatDescription: newFormatDescription,
sampleTiming: &sampleTimingInfo,
sampleBufferOut: &newSampleBuffer
)
if status == noErr, let newSampleBuffer {
self.semaphore.signal()
return newSampleBuffer
}
self.semaphore.signal()
return nil
}
private var currentPosition: Camera.Position = .front
private var lastSwitchTimestamp: Double = 0.0
func markPositionChange(position: Camera.Position) {
self.currentPosition = position
self.lastSwitchTimestamp = CACurrentMediaTime()
if let videoRecorder = self.videoRecorder {
videoRecorder.markPositionChange(position: position)
}
}
}
extension CameraOutput: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard CMSampleBufferDataIsReady(sampleBuffer) else {
return
}
if let videoPixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
self.processSampleBuffer?(sampleBuffer, videoPixelBuffer, connection)
} else if sampleBuffer.type == kCMMediaType_Audio {
self.processAudioBuffer?(sampleBuffer)
}
if let masterOutput = self.masterOutput {
masterOutput.processVideoRecording(sampleBuffer, fromAdditionalOutput: true)
} else {
self.processVideoRecording(sampleBuffer, fromAdditionalOutput: false)
}
}
func captureOutput(_ output: AVCaptureOutput, didDrop sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
if #available(iOS 13.0, *) {
Logger.shared.log("VideoRecorder", "Dropped sample buffer \(sampleBuffer.attachments)")
}
}
}
extension CameraOutput: AVCaptureMetadataOutputObjectsDelegate {
func metadataOutput(_ output: AVCaptureMetadataOutput, didOutput metadataObjects: [AVMetadataObject], from connection: AVCaptureConnection) {
let codes: [CameraCode] = metadataObjects.filter { $0.type == .qr }.compactMap { object in
if let object = object as? AVMetadataMachineReadableCodeObject, let stringValue = object.stringValue, !stringValue.isEmpty {
#if targetEnvironment(simulator)
return CameraCode(type: .qr, message: stringValue, corners: [CGPoint(), CGPoint(), CGPoint(), CGPoint()])
#else
return CameraCode(type: .qr, message: stringValue, corners: object.corners)
#endif
} else {
return nil
}
}
self.processCodes?(codes)
}
}
private let hasHEVCHardwareEncoder: Bool = {
let spec: [CFString: Any] = [:]
var outID: CFString?
var properties: CFDictionary?
let result = VTCopySupportedPropertyDictionaryForEncoder(width: 1920, height: 1080, codecType: kCMVideoCodecType_HEVC, encoderSpecification: spec as CFDictionary, encoderIDOut: &outID, supportedPropertiesOut: &properties)
if result == kVTCouldNotFindVideoEncoderErr {
return false
}
return result == noErr
}()
@@ -0,0 +1,677 @@
import Foundation
import UIKit
import Display
import AVFoundation
import SwiftSignalKit
import Metal
import MetalKit
import CoreMedia
import Vision
import ImageBlur
private extension UIInterfaceOrientation {
var videoOrientation: AVCaptureVideoOrientation {
switch self {
case .portraitUpsideDown: return .portraitUpsideDown
case .landscapeRight: return .landscapeRight
case .landscapeLeft: return .landscapeLeft
case .portrait: return .portrait
default: return .portrait
}
}
}
private class SimpleCapturePreviewLayer: AVCaptureVideoPreviewLayer {
public var didEnterHierarchy: (() -> Void)?
public var didExitHierarchy: (() -> Void)?
override open func action(forKey event: String) -> CAAction? {
if event == kCAOnOrderIn {
self.didEnterHierarchy?()
} else if event == kCAOnOrderOut {
self.didExitHierarchy?()
}
return nullAction
}
override public init(layer: Any) {
super.init(layer: layer)
}
required public init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
}
public class CameraSimplePreviewView: UIView {
func updateOrientation() {
guard self.videoPreviewLayer.connection?.isVideoOrientationSupported == true else {
return
}
let statusBarOrientation: UIInterfaceOrientation
if #available(iOS 13.0, *) {
statusBarOrientation = UIApplication.shared.windows.first?.windowScene?.interfaceOrientation ?? .portrait
} else {
statusBarOrientation = UIApplication.shared.statusBarOrientation
}
let videoOrientation = statusBarOrientation.videoOrientation
self.videoPreviewLayer.connection?.videoOrientation = videoOrientation
self.videoPreviewLayer.removeAllAnimations()
}
static func lastBackImage() -> UIImage {
let imagePath = NSTemporaryDirectory() + "backCameraImage.jpg"
if let data = try? Data(contentsOf: URL(fileURLWithPath: imagePath)), let image = UIImage(data: data) {
return image
} else {
return UIImage(bundleImageName: "Camera/Placeholder")!
}
}
static func saveLastBackImage(_ image: UIImage) {
let imagePath = NSTemporaryDirectory() + "backCameraImage.jpg"
if let data = image.jpegData(compressionQuality: 0.6) {
try? data.write(to: URL(fileURLWithPath: imagePath))
}
}
static func lastFrontImage() -> UIImage {
let imagePath = NSTemporaryDirectory() + "frontCameraImage.jpg"
if let data = try? Data(contentsOf: URL(fileURLWithPath: imagePath)), let image = UIImage(data: data) {
return image
} else {
return UIImage(bundleImageName: "Camera/SelfiePlaceholder")!
}
}
static func saveLastFrontImage(_ image: UIImage) {
let imagePath = NSTemporaryDirectory() + "frontCameraImage.jpg"
if let data = image.jpegData(compressionQuality: 0.6) {
try? data.write(to: URL(fileURLWithPath: imagePath))
}
}
private var previewingDisposable: Disposable?
private let placeholderView = UIImageView()
public init(frame: CGRect, main: Bool, roundVideo: Bool = false) {
super.init(frame: frame)
if roundVideo {
self.videoPreviewLayer.videoGravity = .resizeAspectFill
self.placeholderView.contentMode = .scaleAspectFill
} else {
self.videoPreviewLayer.videoGravity = main ? .resizeAspectFill : .resizeAspect
self.placeholderView.contentMode = main ? .scaleAspectFill : .scaleAspectFit
}
self.addSubview(self.placeholderView)
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
deinit {
self.previewingDisposable?.dispose()
}
public override func layoutSubviews() {
super.layoutSubviews()
self.updateOrientation()
self.placeholderView.frame = self.bounds.insetBy(dx: -1.0, dy: -1.0)
}
public func removePlaceholder(delay: Double = 0.0) {
UIView.animate(withDuration: 0.3, delay: delay) {
self.placeholderView.alpha = 0.0
}
}
public func resetPlaceholder(front: Bool) {
self.placeholderView.image = front ? CameraSimplePreviewView.lastFrontImage() : CameraSimplePreviewView.lastBackImage()
self.placeholderView.alpha = 1.0
}
private var _videoPreviewLayer: AVCaptureVideoPreviewLayer?
var videoPreviewLayer: AVCaptureVideoPreviewLayer {
if let layer = self._videoPreviewLayer {
return layer
}
guard let layer = self.layer as? AVCaptureVideoPreviewLayer else {
fatalError()
}
self._videoPreviewLayer = layer
return layer
}
func invalidate() {
self.videoPreviewLayer.session = nil
}
func setSession(_ session: AVCaptureSession, autoConnect: Bool) {
if autoConnect {
self.videoPreviewLayer.session = session
} else {
self.videoPreviewLayer.setSessionWithNoConnection(session)
}
}
public var isEnabled: Bool = true {
didSet {
self.videoPreviewLayer.connection?.isEnabled = self.isEnabled
}
}
public override class var layerClass: AnyClass {
return AVCaptureVideoPreviewLayer.self
}
@available(iOS 13.0, *)
public var isPreviewing: Signal<Bool, NoError> {
return Signal { [weak self] subscriber in
guard let self else {
return EmptyDisposable
}
subscriber.putNext(self.videoPreviewLayer.isPreviewing)
let observer = self.videoPreviewLayer.observe(\.isPreviewing, options: [.new], changeHandler: { view, _ in
subscriber.putNext(view.isPreviewing)
})
return ActionDisposable {
observer.invalidate()
}
}
|> distinctUntilChanged
}
public func cameraPoint(for location: CGPoint) -> CGPoint {
return self.videoPreviewLayer.captureDevicePointConverted(fromLayerPoint: location)
}
}
public class CameraPreviewView: MTKView {
private let queue = DispatchQueue(label: "CameraPreview", qos: .userInitiated, attributes: [], autoreleaseFrequency: .workItem)
private let commandQueue: MTLCommandQueue
private var textureCache: CVMetalTextureCache?
private var sampler: MTLSamplerState!
private var renderPipelineState: MTLRenderPipelineState!
private var vertexCoordBuffer: MTLBuffer!
private var texCoordBuffer: MTLBuffer!
private var textureWidth: Int = 0
private var textureHeight: Int = 0
private var textureMirroring = false
private var textureRotation: Rotation = .rotate0Degrees
private var textureTranform: CGAffineTransform?
private var _bounds = CGRectNull
public enum Rotation: Int {
case rotate0Degrees
case rotate90Degrees
case rotate180Degrees
case rotate270Degrees
}
private var _mirroring: Bool?
private var _scheduledMirroring: Bool?
public var mirroring = false {
didSet {
self.queue.sync {
if self._mirroring != nil {
self._scheduledMirroring = self.mirroring
} else {
self._mirroring = self.mirroring
}
}
}
}
private var _rotation: Rotation = .rotate0Degrees
public var rotation: Rotation = .rotate0Degrees {
didSet {
self.queue.sync {
self._rotation = rotation
}
}
}
private var _pixelBuffer: CVPixelBuffer?
var pixelBuffer: CVPixelBuffer? {
didSet {
self.queue.sync {
if let scheduledMirroring = self._scheduledMirroring {
self._scheduledMirroring = nil
self._mirroring = scheduledMirroring
}
self._pixelBuffer = pixelBuffer
}
}
}
public init?(test: Bool) {
let mainBundle = Bundle(for: CameraPreviewView.self)
guard let path = mainBundle.path(forResource: "CameraBundle", ofType: "bundle") else {
return nil
}
guard let bundle = Bundle(path: path) else {
return nil
}
guard let device = MTLCreateSystemDefaultDevice() else {
return nil
}
guard let defaultLibrary = try? device.makeDefaultLibrary(bundle: bundle) else {
return nil
}
guard let commandQueue = device.makeCommandQueue() else {
return nil
}
self.commandQueue = commandQueue
super.init(frame: .zero, device: device)
self.colorPixelFormat = .bgra8Unorm
let pipelineDescriptor = MTLRenderPipelineDescriptor()
pipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm
pipelineDescriptor.vertexFunction = defaultLibrary.makeFunction(name: "vertexPassThrough")
pipelineDescriptor.fragmentFunction = defaultLibrary.makeFunction(name: "fragmentPassThrough")
let samplerDescriptor = MTLSamplerDescriptor()
samplerDescriptor.sAddressMode = .clampToEdge
samplerDescriptor.tAddressMode = .clampToEdge
samplerDescriptor.minFilter = .linear
samplerDescriptor.magFilter = .linear
self.sampler = device.makeSamplerState(descriptor: samplerDescriptor)
do {
self.renderPipelineState = try device.makeRenderPipelineState(descriptor: pipelineDescriptor)
} catch {
fatalError("\(error)")
}
self.setupTextureCache()
}
required public init(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
private func setupTextureCache() {
var newTextureCache: CVMetalTextureCache?
if CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device!, nil, &newTextureCache) == kCVReturnSuccess {
self.textureCache = newTextureCache
} else {
assertionFailure("Unable to allocate texture cache")
}
}
private func setupTransform(width: Int, height: Int, rotation: Rotation, mirroring: Bool) {
var scaleX: Float = 1.0
var scaleY: Float = 1.0
var resizeAspect: Float = 1.0
self._bounds = self.bounds
self.textureWidth = width
self.textureHeight = height
self.textureMirroring = mirroring
self.textureRotation = rotation
if self.textureWidth > 0 && self.textureHeight > 0 {
switch self.textureRotation {
case .rotate0Degrees, .rotate180Degrees:
scaleX = Float(self._bounds.width / CGFloat(self.textureWidth))
scaleY = Float(self._bounds.height / CGFloat(self.textureHeight))
case .rotate90Degrees, .rotate270Degrees:
scaleX = Float(self._bounds.width / CGFloat(self.textureHeight))
scaleY = Float(self._bounds.height / CGFloat(self.textureWidth))
}
}
resizeAspect = min(scaleX, scaleY)
if scaleX < scaleY {
scaleY = scaleX / scaleY
scaleX = 1.0
} else {
scaleX = scaleY / scaleX
scaleY = 1.0
}
if self.textureMirroring {
scaleX *= -1.0
}
let vertexData: [Float] = [
-scaleX, -scaleY, 0.0, 1.0,
scaleX, -scaleY, 0.0, 1.0,
-scaleX, scaleY, 0.0, 1.0,
scaleX, scaleY, 0.0, 1.0
]
self.vertexCoordBuffer = device!.makeBuffer(bytes: vertexData, length: vertexData.count * MemoryLayout<Float>.size, options: [])
var texCoordBufferData: [Float]
switch self.textureRotation {
case .rotate0Degrees:
texCoordBufferData = [
0.0, 1.0,
1.0, 1.0,
0.0, 0.0,
1.0, 0.0
]
case .rotate180Degrees:
texCoordBufferData = [
1.0, 0.0,
0.0, 0.0,
1.0, 1.0,
0.0, 1.0
]
case .rotate90Degrees:
texCoordBufferData = [
1.0, 1.0,
1.0, 0.0,
0.0, 1.0,
0.0, 0.0
]
case .rotate270Degrees:
texCoordBufferData = [
0.0, 0.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0
]
}
self.texCoordBuffer = device?.makeBuffer(bytes: texCoordBufferData, length: texCoordBufferData.count * MemoryLayout<Float>.size, options: [])
var transform = CGAffineTransform.identity
if self.textureMirroring {
transform = transform.concatenating(CGAffineTransform(scaleX: -1, y: 1))
transform = transform.concatenating(CGAffineTransform(translationX: CGFloat(self.textureWidth), y: 0))
}
switch self.textureRotation {
case .rotate0Degrees:
transform = transform.concatenating(CGAffineTransform(rotationAngle: CGFloat(0)))
case .rotate180Degrees:
transform = transform.concatenating(CGAffineTransform(rotationAngle: CGFloat(Double.pi)))
transform = transform.concatenating(CGAffineTransform(translationX: CGFloat(self.textureWidth), y: CGFloat(self.textureHeight)))
case .rotate90Degrees:
transform = transform.concatenating(CGAffineTransform(rotationAngle: CGFloat(Double.pi) / 2))
transform = transform.concatenating(CGAffineTransform(translationX: CGFloat(self.textureHeight), y: 0))
case .rotate270Degrees:
transform = transform.concatenating(CGAffineTransform(rotationAngle: 3 * CGFloat(Double.pi) / 2))
transform = transform.concatenating(CGAffineTransform(translationX: 0, y: CGFloat(self.textureWidth)))
}
transform = transform.concatenating(CGAffineTransform(scaleX: CGFloat(resizeAspect), y: CGFloat(resizeAspect)))
let tranformRect = CGRect(origin: .zero, size: CGSize(width: self.textureWidth, height: self.textureHeight)).applying(transform)
let xShift = (self._bounds.size.width - tranformRect.size.width) / 2
let yShift = (self._bounds.size.height - tranformRect.size.height) / 2
transform = transform.concatenating(CGAffineTransform(translationX: xShift, y: yShift))
self.textureTranform = transform.inverted()
}
public override func draw(_ rect: CGRect) {
var pixelBuffer: CVPixelBuffer?
var mirroring = false
var rotation: Rotation = .rotate0Degrees
self.queue.sync {
pixelBuffer = self._pixelBuffer
if let mirroringValue = self._mirroring {
mirroring = mirroringValue
}
rotation = self._rotation
}
guard let drawable = currentDrawable, let currentRenderPassDescriptor = currentRenderPassDescriptor, let previewPixelBuffer = pixelBuffer else {
return
}
let width = CVPixelBufferGetWidth(previewPixelBuffer)
let height = CVPixelBufferGetHeight(previewPixelBuffer)
if self.textureCache == nil {
self.setupTextureCache()
}
var cvTextureOut: CVMetalTexture?
CVMetalTextureCacheCreateTextureFromImage(
kCFAllocatorDefault,
textureCache!,
previewPixelBuffer,
nil,
.bgra8Unorm,
width,
height,
0,
&cvTextureOut)
guard let cvTexture = cvTextureOut, let texture = CVMetalTextureGetTexture(cvTexture) else {
CVMetalTextureCacheFlush(self.textureCache!, 0)
return
}
if texture.width != self.textureWidth ||
texture.height != self.textureHeight ||
self.bounds != self._bounds ||
rotation != self.textureRotation ||
mirroring != self.textureMirroring {
self.setupTransform(width: texture.width, height: texture.height, rotation: rotation, mirroring: mirroring)
}
guard let commandBuffer = self.commandQueue.makeCommandBuffer() else {
CVMetalTextureCacheFlush(self.textureCache!, 0)
return
}
guard let commandEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor) else {
CVMetalTextureCacheFlush(self.textureCache!, 0)
return
}
commandEncoder.setRenderPipelineState(self.renderPipelineState!)
commandEncoder.setVertexBuffer(self.vertexCoordBuffer, offset: 0, index: 0)
commandEncoder.setVertexBuffer(self.texCoordBuffer, offset: 0, index: 1)
commandEncoder.setFragmentTexture(texture, index: 0)
commandEncoder.setFragmentSamplerState(self.sampler, index: 0)
commandEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
commandEncoder.endEncoding()
commandBuffer.present(drawable)
commandBuffer.commit()
}
var captureDeviceResolution: CGSize = CGSize() {
didSet {
if oldValue.width.isZero, !self.captureDeviceResolution.width.isZero {
Queue.mainQueue().async {
self.setupVisionDrawingLayers()
}
}
}
}
var detectionOverlayLayer: CALayer?
var detectedFaceRectangleShapeLayer: CAShapeLayer?
var detectedFaceLandmarksShapeLayer: CAShapeLayer?
func drawFaceObservations(_ faceObservations: [VNFaceObservation]) {
guard let faceRectangleShapeLayer = self.detectedFaceRectangleShapeLayer,
let faceLandmarksShapeLayer = self.detectedFaceLandmarksShapeLayer
else {
return
}
CATransaction.begin()
CATransaction.setValue(NSNumber(value: true), forKey: kCATransactionDisableActions)
self.detectionOverlayLayer?.isHidden = faceObservations.isEmpty
let faceRectanglePath = CGMutablePath()
let faceLandmarksPath = CGMutablePath()
for faceObservation in faceObservations {
self.addIndicators(to: faceRectanglePath,
faceLandmarksPath: faceLandmarksPath,
for: faceObservation)
}
faceRectangleShapeLayer.path = faceRectanglePath
faceLandmarksShapeLayer.path = faceLandmarksPath
self.updateLayerGeometry()
CATransaction.commit()
}
fileprivate func addPoints(in landmarkRegion: VNFaceLandmarkRegion2D, to path: CGMutablePath, applying affineTransform: CGAffineTransform, closingWhenComplete closePath: Bool) {
let pointCount = landmarkRegion.pointCount
if pointCount > 1 {
let points: [CGPoint] = landmarkRegion.normalizedPoints
path.move(to: points[0], transform: affineTransform)
path.addLines(between: points, transform: affineTransform)
if closePath {
path.addLine(to: points[0], transform: affineTransform)
path.closeSubpath()
}
}
}
fileprivate func addIndicators(to faceRectanglePath: CGMutablePath, faceLandmarksPath: CGMutablePath, for faceObservation: VNFaceObservation) {
let displaySize = self.captureDeviceResolution
let faceBounds = VNImageRectForNormalizedRect(faceObservation.boundingBox, Int(displaySize.width), Int(displaySize.height))
faceRectanglePath.addRect(faceBounds)
if let landmarks = faceObservation.landmarks {
let affineTransform = CGAffineTransform(translationX: faceBounds.origin.x, y: faceBounds.origin.y)
.scaledBy(x: faceBounds.size.width, y: faceBounds.size.height)
let openLandmarkRegions: [VNFaceLandmarkRegion2D?] = [
landmarks.leftEyebrow,
landmarks.rightEyebrow,
landmarks.faceContour,
landmarks.noseCrest,
landmarks.medianLine
]
for openLandmarkRegion in openLandmarkRegions where openLandmarkRegion != nil {
self.addPoints(in: openLandmarkRegion!, to: faceLandmarksPath, applying: affineTransform, closingWhenComplete: false)
}
let closedLandmarkRegions: [VNFaceLandmarkRegion2D?] = [
landmarks.leftEye,
landmarks.rightEye,
landmarks.outerLips,
landmarks.innerLips,
landmarks.nose
]
for closedLandmarkRegion in closedLandmarkRegions where closedLandmarkRegion != nil {
self.addPoints(in: closedLandmarkRegion!, to: faceLandmarksPath, applying: affineTransform, closingWhenComplete: true)
}
}
}
fileprivate func radiansForDegrees(_ degrees: CGFloat) -> CGFloat {
return CGFloat(Double(degrees) * Double.pi / 180.0)
}
fileprivate func updateLayerGeometry() {
guard let overlayLayer = self.detectionOverlayLayer else {
return
}
CATransaction.setValue(NSNumber(value: true), forKey: kCATransactionDisableActions)
let videoPreviewRect = self.bounds
var rotation: CGFloat
var scaleX: CGFloat
var scaleY: CGFloat
switch UIDevice.current.orientation {
case .portraitUpsideDown:
rotation = 180
scaleX = videoPreviewRect.width / captureDeviceResolution.width
scaleY = videoPreviewRect.height / captureDeviceResolution.height
case .landscapeLeft:
rotation = 90
scaleX = videoPreviewRect.height / captureDeviceResolution.width
scaleY = scaleX
case .landscapeRight:
rotation = -90
scaleX = videoPreviewRect.height / captureDeviceResolution.width
scaleY = scaleX
default:
rotation = 0
scaleX = videoPreviewRect.width / captureDeviceResolution.width
scaleY = videoPreviewRect.height / captureDeviceResolution.height
}
let affineTransform = CGAffineTransform(rotationAngle: radiansForDegrees(rotation))
.scaledBy(x: scaleX, y: -scaleY)
overlayLayer.setAffineTransform(affineTransform)
let rootLayerBounds = self.bounds
overlayLayer.position = CGPoint(x: rootLayerBounds.midX, y: rootLayerBounds.midY)
}
fileprivate func setupVisionDrawingLayers() {
let captureDeviceResolution = self.captureDeviceResolution
let rootLayer = self.layer
let captureDeviceBounds = CGRect(x: 0,
y: 0,
width: captureDeviceResolution.width,
height: captureDeviceResolution.height)
let captureDeviceBoundsCenterPoint = CGPoint(x: captureDeviceBounds.midX,
y: captureDeviceBounds.midY)
let normalizedCenterPoint = CGPoint(x: 0.5, y: 0.5)
let overlayLayer = CALayer()
overlayLayer.name = "DetectionOverlay"
overlayLayer.masksToBounds = true
overlayLayer.anchorPoint = normalizedCenterPoint
overlayLayer.bounds = captureDeviceBounds
overlayLayer.position = CGPoint(x: rootLayer.bounds.midX, y: rootLayer.bounds.midY)
let faceRectangleShapeLayer = CAShapeLayer()
faceRectangleShapeLayer.name = "RectangleOutlineLayer"
faceRectangleShapeLayer.bounds = captureDeviceBounds
faceRectangleShapeLayer.anchorPoint = normalizedCenterPoint
faceRectangleShapeLayer.position = captureDeviceBoundsCenterPoint
faceRectangleShapeLayer.fillColor = nil
faceRectangleShapeLayer.strokeColor = UIColor.green.withAlphaComponent(0.2).cgColor
faceRectangleShapeLayer.lineWidth = 2
let faceLandmarksShapeLayer = CAShapeLayer()
faceLandmarksShapeLayer.name = "FaceLandmarksLayer"
faceLandmarksShapeLayer.bounds = captureDeviceBounds
faceLandmarksShapeLayer.anchorPoint = normalizedCenterPoint
faceLandmarksShapeLayer.position = captureDeviceBoundsCenterPoint
faceLandmarksShapeLayer.fillColor = nil
faceLandmarksShapeLayer.strokeColor = UIColor.white.withAlphaComponent(0.7).cgColor
faceLandmarksShapeLayer.lineWidth = 2
faceLandmarksShapeLayer.shadowOpacity = 0.7
faceLandmarksShapeLayer.shadowRadius = 2
overlayLayer.addSublayer(faceRectangleShapeLayer)
faceRectangleShapeLayer.addSublayer(faceLandmarksShapeLayer)
self.layer.addSublayer(overlayLayer)
self.detectionOverlayLayer = overlayLayer
self.detectedFaceRectangleShapeLayer = faceRectangleShapeLayer
self.detectedFaceLandmarksShapeLayer = faceLandmarksShapeLayer
self.updateLayerGeometry()
}
}
@@ -0,0 +1,168 @@
import Foundation
import UIKit
import AVFoundation
import CoreImage
import CoreMedia
import CoreVideo
import Metal
import Display
import TelegramCore
final class CameraRoundLegacyVideoFilter {
private let ciContext: CIContext
private let colorSpace: CGColorSpace
private let simple: Bool
private var resizeFilter: CIFilter?
private var overlayFilter: CIFilter?
private var compositeFilter: CIFilter?
private var borderFilter: CIFilter?
private var outputColorSpace: CGColorSpace?
private var outputPixelBufferPool: CVPixelBufferPool?
private(set) var outputFormatDescription: CMFormatDescription?
private(set) var inputFormatDescription: CMFormatDescription?
private(set) var isPrepared = false
init(ciContext: CIContext, colorSpace: CGColorSpace, simple: Bool) {
self.ciContext = ciContext
self.colorSpace = colorSpace
self.simple = simple
}
func prepare(with formatDescription: CMFormatDescription, outputRetainedBufferCountHint: Int) {
self.reset()
(self.outputPixelBufferPool, self.outputColorSpace, self.outputFormatDescription) = allocateOutputBufferPool(with: formatDescription, outputRetainedBufferCountHint: outputRetainedBufferCountHint)
if self.outputPixelBufferPool == nil {
return
}
self.inputFormatDescription = formatDescription
let circleImage = generateImage(videoMessageDimensions.cgSize, opaque: false, scale: 1.0, rotatedContext: { size, context in
let bounds = CGRect(origin: .zero, size: size)
context.clear(bounds)
context.setFillColor(UIColor.white.cgColor)
context.fill(bounds)
context.setBlendMode(.clear)
context.fillEllipse(in: bounds.insetBy(dx: -2.0, dy: -2.0))
})!
self.resizeFilter = CIFilter(name: "CILanczosScaleTransform")
self.overlayFilter = CIFilter(name: "CIColorMatrix")
self.compositeFilter = CIFilter(name: "CISourceOverCompositing")
self.borderFilter = CIFilter(name: "CISourceOverCompositing")
self.borderFilter?.setValue(CIImage(image: circleImage), forKey: kCIInputImageKey)
self.isPrepared = true
}
func reset() {
self.resizeFilter = nil
self.overlayFilter = nil
self.compositeFilter = nil
self.borderFilter = nil
self.outputColorSpace = nil
self.outputPixelBufferPool = nil
self.outputFormatDescription = nil
self.inputFormatDescription = nil
self.isPrepared = false
self.lastMainSourceImage = nil
self.lastAdditionalSourceImage = nil
}
private var lastMainSourceImage: CIImage?
private var lastAdditionalSourceImage: CIImage?
func render(pixelBuffer: CVPixelBuffer, additional: Bool, captureOrientation: AVCaptureVideoOrientation, transitionFactor: CGFloat) -> CVPixelBuffer? {
guard let resizeFilter = self.resizeFilter, let overlayFilter = self.overlayFilter, let compositeFilter = self.compositeFilter, let borderFilter = self.borderFilter, self.isPrepared else {
return nil
}
var sourceImage = CIImage(cvImageBuffer: pixelBuffer, options: [.colorSpace: self.colorSpace])
var sourceOrientation: CGImagePropertyOrientation
var sourceIsLandscape = false
switch captureOrientation {
case .portrait:
sourceOrientation = additional ? .leftMirrored : .right
case .landscapeLeft:
sourceOrientation = additional ? .upMirrored : .down
sourceIsLandscape = true
case .landscapeRight:
sourceOrientation = additional ? .downMirrored : .up
sourceIsLandscape = true
case .portraitUpsideDown:
sourceOrientation = additional ? .rightMirrored : .left
@unknown default:
sourceOrientation = additional ? .leftMirrored : .right
}
sourceImage = sourceImage.oriented(sourceOrientation)
let scale = CGFloat(videoMessageDimensions.width) / min(sourceImage.extent.width, sourceImage.extent.height)
if !self.simple {
resizeFilter.setValue(sourceImage, forKey: kCIInputImageKey)
resizeFilter.setValue(scale, forKey: kCIInputScaleKey)
if let resizedImage = resizeFilter.outputImage {
sourceImage = resizedImage
} else {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeScale(scale, scale), highQualityDownsample: true)
}
} else {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeScale(scale, scale), highQualityDownsample: true)
}
if sourceIsLandscape {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeTranslation(-(sourceImage.extent.width - sourceImage.extent.height) / 2.0, 0.0))
sourceImage = sourceImage.cropped(to: CGRect(x: 0.0, y: 0.0, width: sourceImage.extent.height, height: sourceImage.extent.height))
} else {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeTranslation(0.0, -(sourceImage.extent.height - sourceImage.extent.width) / 2.0))
sourceImage = sourceImage.cropped(to: CGRect(x: 0.0, y: 0.0, width: sourceImage.extent.width, height: sourceImage.extent.width))
}
if additional {
self.lastAdditionalSourceImage = sourceImage
} else {
self.lastMainSourceImage = sourceImage
}
var effectiveSourceImage: CIImage
if transitionFactor == 0.0 {
effectiveSourceImage = !additional ? sourceImage : (self.lastMainSourceImage ?? sourceImage)
} else if transitionFactor == 1.0 {
effectiveSourceImage = additional ? sourceImage : (self.lastAdditionalSourceImage ?? sourceImage)
} else {
if let mainSourceImage = self.lastMainSourceImage, let additionalSourceImage = self.lastAdditionalSourceImage {
let overlayRgba: [CGFloat] = [0, 0, 0, transitionFactor]
let alphaVector: CIVector = CIVector(values: overlayRgba, count: 4)
overlayFilter.setValue(additionalSourceImage, forKey: kCIInputImageKey)
overlayFilter.setValue(alphaVector, forKey: "inputAVector")
compositeFilter.setValue(mainSourceImage, forKey: kCIInputBackgroundImageKey)
compositeFilter.setValue(overlayFilter.outputImage, forKey: kCIInputImageKey)
effectiveSourceImage = compositeFilter.outputImage ?? sourceImage
} else {
effectiveSourceImage = sourceImage
}
}
borderFilter.setValue(effectiveSourceImage, forKey: kCIInputBackgroundImageKey)
let finalImage = borderFilter.outputImage
guard let finalImage else {
return nil
}
var pbuf: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, outputPixelBufferPool!, &pbuf)
guard let outputPixelBuffer = pbuf else {
return nil
}
self.ciContext.render(finalImage, to: outputPixelBuffer, bounds: CGRect(origin: .zero, size: videoMessageDimensions.cgSize), colorSpace: outputColorSpace)
return outputPixelBuffer
}
}
@@ -0,0 +1,352 @@
import Foundation
import UIKit
import AVFoundation
import CoreImage
import CoreMedia
import CoreVideo
import Metal
import Display
import TelegramCore
import RLottieBinding
import GZip
import AppBundle
let videoMessageDimensions = PixelDimensions(width: 400, height: 400)
func allocateOutputBufferPool(with inputFormatDescription: CMFormatDescription, outputRetainedBufferCountHint: Int) -> (
outputBufferPool: CVPixelBufferPool?,
outputColorSpace: CGColorSpace?,
outputFormatDescription: CMFormatDescription?) {
let inputMediaSubType = CMFormatDescriptionGetMediaSubType(inputFormatDescription)
if inputMediaSubType != kCVPixelFormatType_32BGRA {
return (nil, nil, nil)
}
let inputDimensions = CMVideoFormatDescriptionGetDimensions(inputFormatDescription)
var pixelBufferAttributes: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: UInt(inputMediaSubType),
kCVPixelBufferWidthKey as String: Int(inputDimensions.width),
kCVPixelBufferHeightKey as String: Int(inputDimensions.height),
kCVPixelBufferIOSurfacePropertiesKey as String: [:] as NSDictionary
]
var cgColorSpace = CGColorSpaceCreateDeviceRGB()
if let inputFormatDescriptionExtension = CMFormatDescriptionGetExtensions(inputFormatDescription) as Dictionary? {
let colorPrimaries = inputFormatDescriptionExtension[kCVImageBufferColorPrimariesKey]
if let colorPrimaries = colorPrimaries {
var colorSpaceProperties: [String: AnyObject] = [kCVImageBufferColorPrimariesKey as String: colorPrimaries]
if let yCbCrMatrix = inputFormatDescriptionExtension[kCVImageBufferYCbCrMatrixKey] {
colorSpaceProperties[kCVImageBufferYCbCrMatrixKey as String] = yCbCrMatrix
}
if let transferFunction = inputFormatDescriptionExtension[kCVImageBufferTransferFunctionKey] {
colorSpaceProperties[kCVImageBufferTransferFunctionKey as String] = transferFunction
}
pixelBufferAttributes[kCVBufferPropagatedAttachmentsKey as String] = colorSpaceProperties
}
if let cvColorspace = inputFormatDescriptionExtension[kCVImageBufferCGColorSpaceKey] {
cgColorSpace = cvColorspace as! CGColorSpace
} else if (colorPrimaries as? String) == (kCVImageBufferColorPrimaries_P3_D65 as String) {
cgColorSpace = CGColorSpace(name: CGColorSpace.displayP3)!
}
}
let poolAttributes = [kCVPixelBufferPoolMinimumBufferCountKey as String: outputRetainedBufferCountHint]
var cvPixelBufferPool: CVPixelBufferPool?
CVPixelBufferPoolCreate(kCFAllocatorDefault, poolAttributes as NSDictionary?, pixelBufferAttributes as NSDictionary?, &cvPixelBufferPool)
guard let pixelBufferPool = cvPixelBufferPool else {
return (nil, nil, nil)
}
preallocateBuffers(pool: pixelBufferPool, allocationThreshold: outputRetainedBufferCountHint)
var pixelBuffer: CVPixelBuffer?
var outputFormatDescription: CMFormatDescription?
let auxAttributes = [kCVPixelBufferPoolAllocationThresholdKey as String: outputRetainedBufferCountHint] as NSDictionary
CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, pixelBufferPool, auxAttributes, &pixelBuffer)
if let pixelBuffer = pixelBuffer {
CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault,
imageBuffer: pixelBuffer,
formatDescriptionOut: &outputFormatDescription)
}
pixelBuffer = nil
return (pixelBufferPool, cgColorSpace, outputFormatDescription)
}
func preallocateBuffers(pool: CVPixelBufferPool, allocationThreshold: Int) {
var pixelBuffers = [CVPixelBuffer]()
var error: CVReturn = kCVReturnSuccess
let auxAttributes = [kCVPixelBufferPoolAllocationThresholdKey as String: allocationThreshold] as NSDictionary
var pixelBuffer: CVPixelBuffer?
while error == kCVReturnSuccess {
error = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault, pool, auxAttributes, &pixelBuffer)
if let pixelBuffer = pixelBuffer {
pixelBuffers.append(pixelBuffer)
}
pixelBuffer = nil
}
pixelBuffers.removeAll()
}
final class CameraRoundVideoFilter {
private let ciContext: CIContext
private let colorSpace: CGColorSpace
private let simple: Bool
private var resizeFilter: CIFilter?
private var overlayFilter: CIFilter?
private var compositeFilter: CIFilter?
private var maskFilter: CIFilter?
private var blurFilter: CIFilter?
private var darkenFilter: CIFilter?
private var logoImageFilter: CIFilter?
private var logoImage: CIImage?
private var animationImageFilter: CIFilter?
private var animationImage: CIImage?
private var animation: LottieInstance?
private var animationFrameIndex: Int32 = 0
private var outputColorSpace: CGColorSpace?
private var outputPixelBufferPool: CVPixelBufferPool?
private(set) var outputFormatDescription: CMFormatDescription?
private(set) var inputFormatDescription: CMFormatDescription?
private(set) var isPrepared = false
init(ciContext: CIContext, colorSpace: CGColorSpace, simple: Bool) {
self.ciContext = ciContext
self.colorSpace = colorSpace
self.simple = simple
}
func prepare(with formatDescription: CMFormatDescription, outputRetainedBufferCountHint: Int) {
self.reset()
(self.outputPixelBufferPool, self.outputColorSpace, self.outputFormatDescription) = allocateOutputBufferPool(with: formatDescription, outputRetainedBufferCountHint: outputRetainedBufferCountHint)
if self.outputPixelBufferPool == nil {
return
}
self.inputFormatDescription = formatDescription
if let logoImage = UIImage(bundleImageName: "Components/RoundVideoCorner") {
self.logoImage = CIImage(image: logoImage)
}
if let path = getAppBundle().path(forResource: "PlaneLogoPlain", ofType: "tgs"), var data = try? Data(contentsOf: URL(fileURLWithPath: path)) {
if let unpackedData = TGGUnzipData(data, 5 * 1024 * 1024) {
data = unpackedData
self.animation = LottieInstance(data: data, fitzModifier: .none, colorReplacements: [:], cacheKey: "")
}
}
let circleMaskImage = generateImage(videoMessageDimensions.cgSize, opaque: false, scale: 1.0, rotatedContext: { size, context in
let bounds = CGRect(origin: .zero, size: size)
context.clear(bounds)
context.setFillColor(UIColor.black.cgColor)
context.fill(bounds)
context.setBlendMode(.normal)
context.setFillColor(UIColor.white.cgColor)
context.fillEllipse(in: bounds.insetBy(dx: -2.0, dy: -2.0))
})!
self.resizeFilter = CIFilter(name: "CILanczosScaleTransform")
self.overlayFilter = CIFilter(name: "CIColorMatrix")
self.compositeFilter = CIFilter(name: "CISourceOverCompositing")
self.maskFilter = CIFilter(name: "CIBlendWithMask")
self.maskFilter?.setValue(CIImage(image: circleMaskImage), forKey: kCIInputMaskImageKey)
self.blurFilter = CIFilter(name: "CIGaussianBlur")
self.blurFilter?.setValue(30.0, forKey: kCIInputRadiusKey)
self.darkenFilter = CIFilter(name: "CIColorMatrix")
let darkenVector = CIVector(x: 0.25, y: 0, z: 0, w: 0)
self.darkenFilter?.setValue(darkenVector, forKey: "inputRVector")
self.darkenFilter?.setValue(darkenVector, forKey: "inputGVector")
self.darkenFilter?.setValue(darkenVector, forKey: "inputBVector")
self.logoImageFilter = CIFilter(name: "CISourceOverCompositing")
self.animationImageFilter = CIFilter(name: "CISourceOverCompositing")
self.isPrepared = true
}
func reset() {
self.resizeFilter = nil
self.overlayFilter = nil
self.compositeFilter = nil
self.maskFilter = nil
self.blurFilter = nil
self.darkenFilter = nil
self.logoImageFilter = nil
self.animationImageFilter = nil
self.outputColorSpace = nil
self.outputPixelBufferPool = nil
self.outputFormatDescription = nil
self.inputFormatDescription = nil
self.isPrepared = false
self.lastMainSourceImage = nil
self.lastAdditionalSourceImage = nil
}
private var lastMainSourceImage: CIImage?
private var lastAdditionalSourceImage: CIImage?
func render(pixelBuffer: CVPixelBuffer, additional: Bool, captureOrientation: AVCaptureVideoOrientation, transitionFactor: CGFloat) -> CVPixelBuffer? {
guard let resizeFilter = self.resizeFilter,
let overlayFilter = self.overlayFilter,
let compositeFilter = self.compositeFilter,
let maskFilter = self.maskFilter,
let blurFilter = self.blurFilter,
let darkenFilter = self.darkenFilter,
let logoImageFilter = self.logoImageFilter,
let animationImageFilter = self.animationImageFilter,
self.isPrepared else {
return nil
}
var sourceImage = CIImage(cvImageBuffer: pixelBuffer, options: [.colorSpace: self.colorSpace])
var sourceOrientation: CGImagePropertyOrientation
var sourceIsLandscape = false
switch captureOrientation {
case .portrait:
sourceOrientation = additional ? .leftMirrored : .right
case .landscapeLeft:
sourceOrientation = additional ? .upMirrored : .down
sourceIsLandscape = true
case .landscapeRight:
sourceOrientation = additional ? .downMirrored : .up
sourceIsLandscape = true
case .portraitUpsideDown:
sourceOrientation = additional ? .rightMirrored : .left
@unknown default:
sourceOrientation = additional ? .leftMirrored : .right
}
sourceImage = sourceImage.oriented(sourceOrientation)
let scale = CGFloat(videoMessageDimensions.width) / min(sourceImage.extent.width, sourceImage.extent.height)
if !self.simple {
resizeFilter.setValue(sourceImage, forKey: kCIInputImageKey)
resizeFilter.setValue(scale, forKey: kCIInputScaleKey)
if let resizedImage = resizeFilter.outputImage {
sourceImage = resizedImage
} else {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeScale(scale, scale), highQualityDownsample: true)
}
} else {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeScale(scale, scale), highQualityDownsample: true)
}
if sourceIsLandscape {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeTranslation(-(sourceImage.extent.width - sourceImage.extent.height) / 2.0, 0.0))
sourceImage = sourceImage.cropped(to: CGRect(x: 0.0, y: 0.0, width: sourceImage.extent.height, height: sourceImage.extent.height))
} else {
sourceImage = sourceImage.transformed(by: CGAffineTransformMakeTranslation(0.0, -(sourceImage.extent.height - sourceImage.extent.width) / 2.0))
sourceImage = sourceImage.cropped(to: CGRect(x: 0.0, y: 0.0, width: sourceImage.extent.width, height: sourceImage.extent.width))
}
if additional {
self.lastAdditionalSourceImage = sourceImage
} else {
self.lastMainSourceImage = sourceImage
}
var effectiveSourceImage: CIImage
if transitionFactor == 0.0 {
effectiveSourceImage = !additional ? sourceImage : (self.lastMainSourceImage ?? sourceImage)
} else if transitionFactor == 1.0 {
effectiveSourceImage = additional ? sourceImage : (self.lastAdditionalSourceImage ?? sourceImage)
} else {
if let mainSourceImage = self.lastMainSourceImage, let additionalSourceImage = self.lastAdditionalSourceImage {
let overlayRgba: [CGFloat] = [0, 0, 0, transitionFactor]
let alphaVector: CIVector = CIVector(values: overlayRgba, count: 4)
overlayFilter.setValue(additionalSourceImage, forKey: kCIInputImageKey)
overlayFilter.setValue(alphaVector, forKey: "inputAVector")
compositeFilter.setValue(mainSourceImage, forKey: kCIInputBackgroundImageKey)
compositeFilter.setValue(overlayFilter.outputImage, forKey: kCIInputImageKey)
effectiveSourceImage = compositeFilter.outputImage ?? sourceImage
} else {
effectiveSourceImage = sourceImage
}
}
let extendedImage = effectiveSourceImage.clampedToExtent()
blurFilter.setValue(extendedImage, forKey: kCIInputImageKey)
let blurredImage = blurFilter.outputImage ?? effectiveSourceImage
let blurredAndCropped = blurredImage.cropped(to: effectiveSourceImage.extent)
darkenFilter.setValue(blurredAndCropped, forKey: kCIInputImageKey)
let darkenedBlurredBackground = darkenFilter.outputImage ?? blurredAndCropped
maskFilter.setValue(effectiveSourceImage, forKey: kCIInputImageKey)
maskFilter.setValue(darkenedBlurredBackground, forKey: kCIInputBackgroundImageKey)
var finalImage = maskFilter.outputImage
guard let maskedImage = finalImage else {
return nil
}
if let logoImage = self.logoImage {
let overlayWidth: CGFloat = 100.0
let xPosition = maskedImage.extent.width - overlayWidth
let yPosition = 0.0
let transformedOverlay = logoImage.transformed(by: CGAffineTransform(translationX: xPosition, y: yPosition))
logoImageFilter.setValue(transformedOverlay, forKey: kCIInputImageKey)
logoImageFilter.setValue(maskedImage, forKey: kCIInputBackgroundImageKey)
finalImage = logoImageFilter.outputImage ?? maskedImage
} else {
finalImage = maskedImage
}
if let animation = self.animation, let renderContext = DrawingContext(size: CGSize(width: 68.0, height: 68.0), scale: 1.0, clear: true) {
animation.renderFrame(with: self.animationFrameIndex, into: renderContext.bytes.assumingMemoryBound(to: UInt8.self), width: Int32(renderContext.size.width * renderContext.scale), height: Int32(renderContext.size.height * renderContext.scale), bytesPerRow: Int32(renderContext.bytesPerRow))
self.animationFrameIndex += 2
if self.animationFrameIndex >= animation.frameCount {
self.animationFrameIndex = 0
}
if let image = renderContext.generateImage(), let animationImage = CIImage(image: image) {
let xPosition = 0.0
let yPosition = 0.0
let transformedOverlay = animationImage.transformed(by: CGAffineTransform(translationX: xPosition, y: yPosition))
animationImageFilter.setValue(transformedOverlay, forKey: kCIInputImageKey)
animationImageFilter.setValue(finalImage, forKey: kCIInputBackgroundImageKey)
finalImage = animationImageFilter.outputImage ?? maskedImage
} else {
finalImage = maskedImage
}
} else {
finalImage = maskedImage
}
guard let finalImage else {
return nil
}
var pbuf: CVPixelBuffer?
CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, outputPixelBufferPool!, &pbuf)
guard let outputPixelBuffer = pbuf else {
return nil
}
self.ciContext.render(finalImage, to: outputPixelBuffer, bounds: CGRect(origin: .zero, size: videoMessageDimensions.cgSize), colorSpace: outputColorSpace)
return outputPixelBuffer
}
}
+367
View File
@@ -0,0 +1,367 @@
import UIKit
import AVFoundation
import Foundation
import Accelerate
import CoreImage
extension AVFrameRateRange {
func clamp(rate: Float64) -> Float64 {
return max(self.minFrameRate, min(self.maxFrameRate, rate))
}
func contains(rate: Float64) -> Bool {
return (self.minFrameRate...self.maxFrameRate) ~= rate
}
}
extension AVCaptureDevice {
func actualFPS(_ fps: Double) -> (fps: Double, duration: CMTime)? {
var durations: [CMTime] = []
var frameRates: [Double] = []
for range in self.activeFormat.videoSupportedFrameRateRanges {
if range.minFrameRate == range.maxFrameRate {
durations.append(range.minFrameDuration)
frameRates.append(range.maxFrameRate)
continue
}
if range.contains(rate: fps) {
return (fps, CMTimeMake(value: 100, timescale: Int32(100 * fps)))
}
let actualFPS: Double = range.clamp(rate: fps)
return (actualFPS, CMTimeMake(value: 100, timescale: Int32(100 * actualFPS)))
}
let diff = frameRates.map { abs($0 - fps) }
if let minElement: Float64 = diff.min() {
for i in 0..<diff.count where diff[i] == minElement {
return (frameRates[i], durations[i])
}
}
return nil
}
var neutralZoomFactor: CGFloat {
if #available(iOS 13.0, *) {
if let indexOfWideAngle = self.constituentDevices.firstIndex(where: { $0.deviceType == .builtInWideAngleCamera }), indexOfWideAngle > 0 {
let zoomFactor = self.virtualDeviceSwitchOverVideoZoomFactors[indexOfWideAngle - 1]
return CGFloat(zoomFactor.doubleValue)
}
}
return 1.0
}
}
extension CMSampleBuffer {
var presentationTimestamp: CMTime {
return CMSampleBufferGetPresentationTimeStamp(self)
}
var type: CMMediaType {
if let formatDescription = CMSampleBufferGetFormatDescription(self) {
return CMFormatDescriptionGetMediaType(formatDescription)
} else {
return kCMMediaType_Video
}
}
}
extension AVCaptureVideoOrientation {
init?(interfaceOrientation: UIInterfaceOrientation) {
switch interfaceOrientation {
case .portrait: self = .portrait
case .portraitUpsideDown: self = .portraitUpsideDown
case .landscapeLeft: self = .landscapeLeft
case .landscapeRight: self = .landscapeRight
default: return nil
}
}
}
extension CameraPreviewView.Rotation {
init?(with interfaceOrientation: UIInterfaceOrientation, videoOrientation: AVCaptureVideoOrientation, cameraPosition: AVCaptureDevice.Position) {
switch videoOrientation {
case .portrait:
switch interfaceOrientation {
case .landscapeRight:
if cameraPosition == .front {
self = .rotate90Degrees
} else {
self = .rotate270Degrees
}
case .landscapeLeft:
if cameraPosition == .front {
self = .rotate270Degrees
} else {
self = .rotate90Degrees
}
case .portrait:
self = .rotate0Degrees
case .portraitUpsideDown:
self = .rotate180Degrees
default: return nil
}
case .portraitUpsideDown:
switch interfaceOrientation {
case .landscapeRight:
if cameraPosition == .front {
self = .rotate270Degrees
} else {
self = .rotate90Degrees
}
case .landscapeLeft:
if cameraPosition == .front {
self = .rotate90Degrees
} else {
self = .rotate270Degrees
}
case .portrait:
self = .rotate180Degrees
case .portraitUpsideDown:
self = .rotate0Degrees
default: return nil
}
case .landscapeRight:
switch interfaceOrientation {
case .landscapeRight:
self = .rotate0Degrees
case .landscapeLeft:
self = .rotate180Degrees
case .portrait:
if cameraPosition == .front {
self = .rotate270Degrees
} else {
self = .rotate90Degrees
}
case .portraitUpsideDown:
if cameraPosition == .front {
self = .rotate90Degrees
} else {
self = .rotate270Degrees
}
default: return nil
}
case .landscapeLeft:
switch interfaceOrientation {
case .landscapeLeft:
self = .rotate0Degrees
case .landscapeRight:
self = .rotate180Degrees
case .portrait:
if cameraPosition == .front {
self = .rotate90Degrees
} else {
self = .rotate270Degrees
}
case .portraitUpsideDown:
if cameraPosition == .front {
self = .rotate270Degrees
} else {
self = .rotate90Degrees
}
default: return nil
}
@unknown default:
fatalError("Unknown orientation.")
}
}
}
func exifOrientation(for orientation: AVCaptureVideoOrientation, mirror: Bool) -> Int32 {
switch (orientation, mirror) {
case (.portrait, false):
return 6
case (.portrait, true):
return 5
case (.portraitUpsideDown, false):
return 8
case (.portraitUpsideDown, true):
return 7
case (.landscapeLeft, false):
return 3
case (.landscapeLeft, true):
return 2
case (.landscapeRight, false):
return 1
case (.landscapeRight, true):
return 4
@unknown default:
return 6
}
}
func resizePixelBuffer(from srcPixelBuffer: CVPixelBuffer,
to dstPixelBuffer: CVPixelBuffer,
cropX: Int,
cropY: Int,
cropWidth: Int,
cropHeight: Int,
scaleWidth: Int,
scaleHeight: Int) {
assert(CVPixelBufferGetWidth(dstPixelBuffer) >= scaleWidth)
assert(CVPixelBufferGetHeight(dstPixelBuffer) >= scaleHeight)
let srcFlags = CVPixelBufferLockFlags.readOnly
let dstFlags = CVPixelBufferLockFlags(rawValue: 0)
guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(srcPixelBuffer, srcFlags) else {
print("Error: could not lock source pixel buffer")
return
}
defer { CVPixelBufferUnlockBaseAddress(srcPixelBuffer, srcFlags) }
guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(dstPixelBuffer, dstFlags) else {
print("Error: could not lock destination pixel buffer")
return
}
defer { CVPixelBufferUnlockBaseAddress(dstPixelBuffer, dstFlags) }
guard let srcData = CVPixelBufferGetBaseAddress(srcPixelBuffer),
let dstData = CVPixelBufferGetBaseAddress(dstPixelBuffer) else {
print("Error: could not get pixel buffer base address")
return
}
let srcBytesPerRow = CVPixelBufferGetBytesPerRow(srcPixelBuffer)
let offset = cropY*srcBytesPerRow + cropX*4
var srcBuffer = vImage_Buffer(data: srcData.advanced(by: offset),
height: vImagePixelCount(cropHeight),
width: vImagePixelCount(cropWidth),
rowBytes: srcBytesPerRow)
let dstBytesPerRow = CVPixelBufferGetBytesPerRow(dstPixelBuffer)
var dstBuffer = vImage_Buffer(data: dstData,
height: vImagePixelCount(scaleHeight),
width: vImagePixelCount(scaleWidth),
rowBytes: dstBytesPerRow)
let error = vImageScale_ARGB8888(&srcBuffer, &dstBuffer, nil, vImage_Flags(0))
if error != kvImageNoError {
print("Error:", error)
}
}
func resizePixelBuffer(from srcPixelBuffer: CVPixelBuffer,
to dstPixelBuffer: CVPixelBuffer,
width: Int, height: Int) {
resizePixelBuffer(from: srcPixelBuffer, to: dstPixelBuffer,
cropX: 0, cropY: 0,
cropWidth: CVPixelBufferGetWidth(srcPixelBuffer),
cropHeight: CVPixelBufferGetHeight(srcPixelBuffer),
scaleWidth: width, scaleHeight: height)
}
func resizePixelBuffer(_ pixelBuffer: CVPixelBuffer,
width: Int, height: Int,
output: CVPixelBuffer, context: CIContext) {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
let sx = CGFloat(width) / CGFloat(CVPixelBufferGetWidth(pixelBuffer))
let sy = CGFloat(height) / CGFloat(CVPixelBufferGetHeight(pixelBuffer))
let scaleTransform = CGAffineTransform(scaleX: sx, y: sy)
let scaledImage = ciImage.transformed(by: scaleTransform)
context.render(scaledImage, to: output)
}
func imageFromCVPixelBuffer(_ pixelBuffer: CVPixelBuffer, orientation: UIImage.Orientation) -> UIImage? {
CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
guard let context = CGContext(
data: baseAddress,
width: width,
height: height,
bitsPerComponent: 8,
bytesPerRow: bytesPerRow,
space: colorSpace,
bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue
) else {
CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
return nil
}
guard let cgImage = context.makeImage() else {
CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
return nil
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
return UIImage(cgImage: cgImage, scale: 1.0, orientation: orientation)
}
extension CVPixelBuffer {
func deepCopy() -> CVPixelBuffer? {
let width = CVPixelBufferGetWidth(self)
let height = CVPixelBufferGetHeight(self)
let format = CVPixelBufferGetPixelFormatType(self)
let attributes: [NSObject: AnyObject] = [
kCVPixelBufferCGImageCompatibilityKey: true as AnyObject,
kCVPixelBufferCGBitmapContextCompatibilityKey: true as AnyObject
]
var newPixelBuffer: CVPixelBuffer?
let status = CVPixelBufferCreate(
kCFAllocatorDefault,
width,
height,
format,
attributes as CFDictionary,
&newPixelBuffer
)
guard status == kCVReturnSuccess, let unwrappedPixelBuffer = newPixelBuffer else {
return nil
}
CVPixelBufferLockBaseAddress(self, .readOnly)
CVPixelBufferLockBaseAddress(unwrappedPixelBuffer, [])
guard let sourceBaseAddress = CVPixelBufferGetBaseAddress(self),
let destinationBaseAddress = CVPixelBufferGetBaseAddress(unwrappedPixelBuffer) else {
CVPixelBufferUnlockBaseAddress(self, .readOnly)
CVPixelBufferUnlockBaseAddress(unwrappedPixelBuffer, [])
return nil
}
let sourceBytesPerRow = CVPixelBufferGetBytesPerRow(self)
let destinationBytesPerRow = CVPixelBufferGetBytesPerRow(unwrappedPixelBuffer)
let imageSize = height * min(sourceBytesPerRow, destinationBytesPerRow)
memcpy(destinationBaseAddress, sourceBaseAddress, imageSize)
CVPixelBufferUnlockBaseAddress(self, .readOnly)
CVPixelBufferUnlockBaseAddress(unwrappedPixelBuffer, [])
return unwrappedPixelBuffer
}
}
@@ -0,0 +1,91 @@
import Foundation
import AVFoundation
import UIKit
import SwiftSignalKit
public enum PhotoCaptureResult: Equatable {
case began
case finished(UIImage, UIImage?, Double)
case failed
public static func == (lhs: PhotoCaptureResult, rhs: PhotoCaptureResult) -> Bool {
switch lhs {
case .began:
if case .began = rhs {
return true
} else {
return false
}
case .failed:
if case .failed = rhs {
return true
} else {
return false
}
case let .finished(_, _, lhsTime):
if case let .finished(_, _, rhsTime) = rhs, lhsTime == rhsTime {
return true
} else {
return false
}
}
}
}
final class PhotoCaptureContext: NSObject, AVCapturePhotoCaptureDelegate {
private let ciContext: CIContext
private let pipe = ValuePipe<PhotoCaptureResult>()
private let orientation: AVCaptureVideoOrientation
private let mirror: Bool
init(ciContext: CIContext, settings: AVCapturePhotoSettings, orientation: AVCaptureVideoOrientation, mirror: Bool) {
self.ciContext = ciContext
self.orientation = orientation
self.mirror = mirror
super.init()
}
func photoOutput(_ output: AVCapturePhotoOutput, willCapturePhotoFor resolvedSettings: AVCaptureResolvedPhotoSettings) {
self.pipe.putNext(.began)
}
func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
if let _ = error {
self.pipe.putNext(.failed)
} else {
guard let photoPixelBuffer = photo.pixelBuffer else {
print("Error occurred while capturing photo: Missing pixel buffer (\(String(describing: error)))")
return
}
//if let value = photo.metadata[kCGImagePropertyOrientation as String] as? NSNumber {
// orientation = value.int32Value
//} else {
let orientation = exifOrientation(for: self.orientation, mirror: self.mirror)
//}
let ci = CIImage(cvImageBuffer: photoPixelBuffer).oriented(forExifOrientation: orientation)
if let cgImage = self.ciContext.createCGImage(ci, from: ci.extent) {
let image = UIImage(cgImage: cgImage, scale: 1.0, orientation: .up)
self.pipe.putNext(.finished(image, nil, CACurrentMediaTime()))
} else {
self.pipe.putNext(.failed)
}
}
}
var signal: Signal<PhotoCaptureResult, NoError> {
return self.pipe.signal()
|> take(until: { next in
let complete: Bool
switch next {
case .finished, .failed:
complete = true
default:
complete = false
}
return SignalTakeAction(passthrough: true, complete: complete)
})
}
}
@@ -0,0 +1,588 @@
import Foundation
import AVFoundation
import UIKit
import CoreImage
import SwiftSignalKit
import TelegramCore
private extension CMSampleBuffer {
var endTime: CMTime {
let presentationTime = CMSampleBufferGetPresentationTimeStamp(self)
let duration = CMSampleBufferGetDuration(self)
return presentationTime + duration
}
}
private final class VideoRecorderImpl {
public enum RecorderError: LocalizedError {
case generic
case avError(Error)
public var errorDescription: String? {
switch self {
case .generic:
return "Error"
case let .avError(error):
return error.localizedDescription
}
}
}
private let queue = DispatchQueue(label: "VideoRecorder")
private var assetWriter: AVAssetWriter
private var videoInput: AVAssetWriterInput?
private var audioInput: AVAssetWriterInput?
private let ciContext: CIContext
fileprivate var transitionImage: UIImage?
private var savedTransitionImage = false
private var pendingAudioSampleBuffers: [CMSampleBuffer] = []
private var _duration = Atomic<CMTime>(value: .zero)
public var duration: CMTime {
return self._duration.with { $0 }
}
private var startedSession = false
private var lastVideoSampleTime: CMTime = .invalid
private var recordingStartSampleTime: CMTime = .invalid
private var recordingStopSampleTime: CMTime = .invalid
private var positionChangeTimestamps: [(Camera.Position, CMTime)] = []
private let configuration: VideoRecorder.Configuration
private let orientation: AVCaptureVideoOrientation
private let videoTransform: CGAffineTransform
private let url: URL
fileprivate var completion: (Bool, UIImage?, [(Camera.Position, CMTime)]?) -> Void = { _, _, _ in }
private let error = Atomic<Error?>(value: nil)
private var _stopped = Atomic<Bool>(value: false)
private var stopped: Bool {
return self._stopped.with { $0 }
}
private var hasAllVideoBuffers = false
private var hasAllAudioBuffers = false
public init?(configuration: VideoRecorder.Configuration, ciContext: CIContext, orientation: AVCaptureVideoOrientation, fileUrl: URL) {
self.configuration = configuration
self.ciContext = ciContext
var transform: CGAffineTransform = CGAffineTransform(rotationAngle: .pi / 2.0)
if orientation == .landscapeLeft {
transform = CGAffineTransform(rotationAngle: .pi)
} else if orientation == .landscapeRight {
transform = CGAffineTransform(rotationAngle: 0.0)
} else if orientation == .portraitUpsideDown {
transform = CGAffineTransform(rotationAngle: -.pi / 2.0)
}
self.orientation = orientation
self.videoTransform = transform
self.url = fileUrl
try? FileManager.default.removeItem(at: url)
guard let assetWriter = try? AVAssetWriter(url: url, fileType: .mp4) else {
return nil
}
self.assetWriter = assetWriter
self.assetWriter.shouldOptimizeForNetworkUse = false
}
private func hasError() -> Error? {
return self.error.with { $0 }
}
public func start() {
self.queue.async {
self.recordingStartSampleTime = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: CMTimeScale(NSEC_PER_SEC))
}
}
public func markPositionChange(position: Camera.Position, time: CMTime? = nil) {
self.queue.async {
guard self.recordingStartSampleTime.isValid || time != nil else {
return
}
if let time {
self.positionChangeTimestamps.append((position, time))
} else {
let currentTime = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: CMTimeScale(NSEC_PER_SEC))
let delta = currentTime - self.recordingStartSampleTime
self.positionChangeTimestamps.append((position, delta))
}
}
}
private var previousPresentationTime: Double?
private var previousAppendTime: Double?
public func appendVideoSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
#if compiler(>=6.0) // Xcode 16
nonisolated(unsafe) let sampleBuffer = sampleBuffer
#endif
self.queue.async {
guard self.hasError() == nil && !self.stopped else {
return
}
guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), CMFormatDescriptionGetMediaType(formatDescription) == kCMMediaType_Video else {
return
}
let presentationTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
var failed = false
if self.videoInput == nil {
Logger.shared.log("VideoRecorder", "Try adding video input")
let videoSettings = self.configuration.videoSettings
if self.assetWriter.canApply(outputSettings: videoSettings, forMediaType: .video) {
let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings, sourceFormatHint: formatDescription)
videoInput.expectsMediaDataInRealTime = true
videoInput.transform = self.videoTransform
if self.assetWriter.canAdd(videoInput) {
self.assetWriter.add(videoInput)
self.videoInput = videoInput
Logger.shared.log("VideoRecorder", "Successfully added video input")
} else {
failed = true
}
} else {
failed = true
}
}
if failed {
Logger.shared.log("VideoRecorder", "Failed to append video buffer")
return
}
if self.assetWriter.status == .unknown {
if sampleBuffer.presentationTimestamp < self.recordingStartSampleTime {
return
}
if self.videoInput != nil && (self.audioInput != nil || !self.configuration.hasAudio) {
print("startWriting")
let start = CACurrentMediaTime()
if !self.assetWriter.startWriting() {
if let error = self.assetWriter.error {
self.transitionToFailedStatus(error: .avError(error))
}
}
print("started In \(CACurrentMediaTime() - start)")
return
}
} else if self.assetWriter.status == .writing && !self.startedSession {
print("Started session at \(presentationTime)")
self.assetWriter.startSession(atSourceTime: presentationTime)
self.recordingStartSampleTime = presentationTime
self.lastVideoSampleTime = presentationTime
self.startedSession = true
}
if self.recordingStartSampleTime == .invalid || sampleBuffer.presentationTimestamp < self.recordingStartSampleTime {
return
}
if self.assetWriter.status == .writing && self.startedSession {
if self.recordingStopSampleTime != .invalid && sampleBuffer.presentationTimestamp > self.recordingStopSampleTime {
self.hasAllVideoBuffers = true
self.maybeFinish()
return
}
if let videoInput = self.videoInput {
while (!videoInput.isReadyForMoreMediaData)
{
let maxDate = Date(timeIntervalSinceNow: 0.05)
RunLoop.current.run(until: maxDate)
}
}
if let videoInput = self.videoInput {
let time = CACurrentMediaTime()
// if let previousPresentationTime = self.previousPresentationTime, let previousAppendTime = self.previousAppendTime {
// print("appending \(presentationTime.seconds) (\(presentationTime.seconds - previousPresentationTime) ) on \(time) (\(time - previousAppendTime)")
// }
self.previousPresentationTime = presentationTime.seconds
self.previousAppendTime = time
if videoInput.append(sampleBuffer) {
self.lastVideoSampleTime = presentationTime
let startTime = self.recordingStartSampleTime
let duration = presentationTime - startTime
let _ = self._duration.modify { _ in return duration }
}
if !self.savedTransitionImage, let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
self.savedTransitionImage = true
Queue.concurrentBackgroundQueue().async {
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
if let cgImage = self.ciContext.createCGImage(ciImage, from: ciImage.extent) {
var orientation: UIImage.Orientation = .right
if self.orientation == .landscapeLeft {
orientation = .down
} else if self.orientation == .landscapeRight {
orientation = .up
} else if self.orientation == .portraitUpsideDown {
orientation = .left
}
self.transitionImage = UIImage(cgImage: cgImage, scale: 1.0, orientation: orientation)
} else {
self.savedTransitionImage = false
}
}
}
if !self.tryAppendingPendingAudioBuffers() {
self.transitionToFailedStatus(error: .generic)
}
}
}
}
}
public func appendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
#if compiler(>=6.0) // Xcode 16
nonisolated(unsafe) let sampleBuffer = sampleBuffer
#endif
self.queue.async {
guard self.hasError() == nil && !self.stopped else {
return
}
guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), CMFormatDescriptionGetMediaType(formatDescription) == kCMMediaType_Audio else {
return
}
var failed = false
if self.audioInput == nil {
Logger.shared.log("VideoRecorder", "Try adding audio input")
var audioSettings = self.configuration.audioSettings
if let currentAudioStreamBasicDescription = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription) {
audioSettings[AVSampleRateKey] = currentAudioStreamBasicDescription.pointee.mSampleRate
audioSettings[AVNumberOfChannelsKey] = currentAudioStreamBasicDescription.pointee.mChannelsPerFrame
}
var audioChannelLayoutSize: Int = 0
let currentChannelLayout = CMAudioFormatDescriptionGetChannelLayout(formatDescription, sizeOut: &audioChannelLayoutSize)
let currentChannelLayoutData: Data
if let currentChannelLayout = currentChannelLayout, audioChannelLayoutSize > 0 {
currentChannelLayoutData = Data(bytes: currentChannelLayout, count: audioChannelLayoutSize)
} else {
currentChannelLayoutData = Data()
}
audioSettings[AVChannelLayoutKey] = currentChannelLayoutData
if self.assetWriter.canApply(outputSettings: audioSettings, forMediaType: .audio) {
let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings, sourceFormatHint: formatDescription)
audioInput.expectsMediaDataInRealTime = true
if self.assetWriter.canAdd(audioInput) {
self.assetWriter.add(audioInput)
self.audioInput = audioInput
Logger.shared.log("VideoRecorder", "Successfully added audio input")
} else {
failed = true
}
} else {
failed = true
}
}
if failed {
Logger.shared.log("VideoRecorder", "Failed to append audio buffer")
return
}
if self.recordingStartSampleTime != .invalid {
if sampleBuffer.presentationTimestamp < self.recordingStartSampleTime {
return
}
if self.recordingStopSampleTime != .invalid && sampleBuffer.presentationTimestamp > self.recordingStopSampleTime {
self.hasAllAudioBuffers = true
self.maybeFinish()
return
}
var result = false
if self.tryAppendingPendingAudioBuffers() {
if self.tryAppendingAudioSampleBuffer(sampleBuffer) {
result = true
}
}
if !result {
self.transitionToFailedStatus(error: .generic)
}
}
}
}
public func cancelRecording(completion: @escaping () -> Void) {
self.queue.async {
if self.stopped {
DispatchQueue.main.async {
completion()
}
return
}
let _ = self._stopped.modify { _ in return true }
self.pendingAudioSampleBuffers = []
if self.assetWriter.status == .writing {
self.assetWriter.cancelWriting()
}
let fileManager = FileManager()
try? fileManager.removeItem(at: self.url)
DispatchQueue.main.async {
completion()
}
}
}
public var isRecording: Bool {
return !self.stopped
}
public func stopRecording() {
self.queue.async {
var stopTime = CMTime(seconds: CACurrentMediaTime(), preferredTimescale: CMTimeScale(NSEC_PER_SEC))
if self.recordingStartSampleTime.isValid {
if (stopTime - self.recordingStartSampleTime).seconds < 1.5 {
stopTime = self.recordingStartSampleTime + CMTime(seconds: 1.5, preferredTimescale: self.recordingStartSampleTime.timescale)
}
}
self.recordingStopSampleTime = stopTime
}
}
private func maybeFinish() {
dispatchPrecondition(condition: .onQueue(self.queue))
guard self.hasAllVideoBuffers && self.hasAllVideoBuffers && !self.stopped else {
return
}
let _ = self._stopped.modify { _ in return true }
self.finish()
}
private func finish() {
dispatchPrecondition(condition: .onQueue(self.queue))
let completion = self.completion
if self.recordingStopSampleTime == .invalid {
DispatchQueue.main.async {
completion(false, nil, nil)
}
return
}
if let _ = self.error.with({ $0 }) {
DispatchQueue.main.async {
completion(false, nil, nil)
}
return
}
if !self.tryAppendingPendingAudioBuffers() {
DispatchQueue.main.async {
completion(false, nil, nil)
}
return
}
if self.assetWriter.status == .writing {
self.assetWriter.finishWriting {
if let _ = self.assetWriter.error {
DispatchQueue.main.async {
completion(false, nil, nil)
}
} else {
DispatchQueue.main.async {
completion(true, self.transitionImage, self.positionChangeTimestamps)
}
}
}
} else if let _ = self.assetWriter.error {
DispatchQueue.main.async {
completion(false, nil, nil)
}
} else {
DispatchQueue.main.async {
completion(false, nil, nil)
}
}
}
private func tryAppendingPendingAudioBuffers() -> Bool {
dispatchPrecondition(condition: .onQueue(self.queue))
guard self.pendingAudioSampleBuffers.count > 0 else {
return true
}
var result = true
let (sampleBuffersToAppend, pendingSampleBuffers) = self.pendingAudioSampleBuffers.stableGroup(using: { $0.endTime <= self.lastVideoSampleTime })
for sampleBuffer in sampleBuffersToAppend {
if !self.internalAppendAudioSampleBuffer(sampleBuffer) {
result = false
break
}
}
self.pendingAudioSampleBuffers = pendingSampleBuffers
return result
}
private func tryAppendingAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer) -> Bool {
dispatchPrecondition(condition: .onQueue(self.queue))
var result = true
if sampleBuffer.endTime > self.lastVideoSampleTime {
self.pendingAudioSampleBuffers.append(sampleBuffer)
} else {
result = self.internalAppendAudioSampleBuffer(sampleBuffer)
}
return result
}
private func internalAppendAudioSampleBuffer(_ sampleBuffer: CMSampleBuffer) -> Bool {
if self.startedSession, let audioInput = self.audioInput {
while (!audioInput.isReadyForMoreMediaData)
{
let maxDate = Date(timeIntervalSinceNow: 0.05)
RunLoop.current.run(until: maxDate)
}
if !audioInput.append(sampleBuffer) {
if let _ = self.assetWriter.error {
return false
}
}
} else {
}
return true
}
private func transitionToFailedStatus(error: RecorderError) {
let _ = self.error.modify({ _ in return error })
}
}
private extension Sequence {
func stableGroup(using predicate: (Element) throws -> Bool) rethrows -> ([Element], [Element]) {
var trueGroup: [Element] = []
var falseGroup: [Element] = []
for element in self {
if try predicate(element) {
trueGroup.append(element)
} else {
falseGroup.append(element)
}
}
return (trueGroup, falseGroup)
}
}
public final class VideoRecorder {
var duration: Double? {
return self.impl.duration.seconds
}
enum Result {
enum Error {
case generic
}
case success(UIImage?, Double, [(Camera.Position, Double)])
case initError(Error)
case writeError(Error)
case finishError(Error)
}
struct Configuration {
var videoSettings: [String: Any]
var audioSettings: [String: Any]
init(videoSettings: [String: Any], audioSettings: [String: Any]) {
self.videoSettings = videoSettings
self.audioSettings = audioSettings
}
var hasAudio: Bool {
return !self.audioSettings.isEmpty
}
}
private let impl: VideoRecorderImpl
fileprivate let configuration: Configuration
fileprivate let fileUrl: URL
private let completion: (Result) -> Void
public var isRecording: Bool {
return self.impl.isRecording
}
init?(configuration: Configuration, ciContext: CIContext, orientation: AVCaptureVideoOrientation, fileUrl: URL, completion: @escaping (Result) -> Void) {
self.configuration = configuration
self.fileUrl = fileUrl
self.completion = completion
guard let impl = VideoRecorderImpl(configuration: configuration, ciContext: ciContext, orientation: orientation, fileUrl: fileUrl) else {
completion(.initError(.generic))
return nil
}
self.impl = impl
impl.completion = { [weak self] result, transitionImage, positionChangeTimestamps in
if let self {
let duration = self.duration ?? 0.0
if result {
var timestamps: [(Camera.Position, Double)] = []
if let positionChangeTimestamps {
for (position, time) in positionChangeTimestamps {
timestamps.append((position, time.seconds))
}
}
self.completion(.success(transitionImage, duration, timestamps))
} else {
self.completion(.finishError(.generic))
}
}
}
}
func start() {
self.impl.start()
}
func stop() {
self.impl.stopRecording()
}
func markPositionChange(position: Camera.Position, time: CMTime? = nil) {
self.impl.markPositionChange(position: position, time: time)
}
func appendSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
guard let formatDescriptor = CMSampleBufferGetFormatDescription(sampleBuffer) else {
return
}
let type = CMFormatDescriptionGetMediaType(formatDescriptor)
if type == kCMMediaType_Video {
self.impl.appendVideoSampleBuffer(sampleBuffer)
} else if type == kCMMediaType_Audio {
if self.configuration.hasAudio {
self.impl.appendAudioSampleBuffer(sampleBuffer)
}
}
}
var transitionImage: UIImage? {
return self.impl.transitionImage
}
}