I have an app that records a 32 x 32 rect under the cursor as the user moves it around and it sends it to Flutter.
It suffers from major lag.
Instead of getting 30 fps, I get about 7 fps. That is, there are significant lags between screen grabs.
This on an Intel Mac mini x64 with 15.7.3 and one display.
flutter: NATIVE: ExplodedView framesIn=2 timeSinceStart=1115.7ms gapSinceLastFrame=838.8ms
flutter: NATIVE: ExplodedView framesIn=4 timeSinceStart=1382.6ms gapSinceLastFrame=149.9ms
flutter: NATIVE: ExplodedView framesIn=5 timeSinceStart=1511.0ms gapSinceLastFrame=128.4ms
flutter: NATIVE: ExplodedView framesIn=7 timeSinceStart=1698.3ms gapSinceLastFrame=102.9ms
flutter: NATIVE: ExplodedView STOP polling totalTime=4482.6ms framesIn=28 framesSent=28 acks=28
Here's a testable excerpt:
import ScreenCaptureKit
import CoreMedia
import CoreVideo
import QuartzCore
final class Test: NSObject, SCStreamOutput, SCStreamDelegate {
private let q = DispatchQueue(label: "cap.q")
private var stream: SCStream?
private var lastFrameAt: CFTimeInterval = 0
private var frames = 0
func start() {
SCShareableContent.getExcludingDesktopWindows(false, onScreenWindowsOnly: true) { content, err in
guard err == nil, let display = content?.displays.first else {
print("shareableContent error: \(String(describing: err))"); return
}
let filter = SCContentFilter(display: display, excludingWindows: [])
let config = SCStreamConfiguration()
config.showsCursor = false
config.queueDepth = 1
config.minimumFrameInterval = CMTime(value: 1, timescale: 30)
config.pixelFormat = kCVPixelFormatType_32BGRA
config.width = 32
config.height = 32
config.sourceRect = CGRect(x: 100, y: 100, width: 32, height: 32)
let s = SCStream(filter: filter, configuration: config, delegate: self)
try! s.addStreamOutput(self, type: .screen, sampleHandlerQueue: self.q)
self.stream = s
s.startCapture { startErr in
print("startCapture err=\(String(describing: startErr))")
}
// Optional: move sourceRect at 30Hz (cursor-follow simulation)
Timer.scheduledTimer(withTimeInterval: 1.0/30.0, repeats: true) { _ in
let c2 = SCStreamConfiguration()
c2.showsCursor = false
c2.queueDepth = 1
c2.minimumFrameInterval = CMTime(value: 1, timescale: 30)
c2.pixelFormat = kCVPixelFormatType_32BGRA
c2.width = 32
c2.height = 32
let t = CACurrentMediaTime()
c2.sourceRect = CGRect(x: 100 + (sin(t) * 50), y: 100, width: 32, height: 32)
s.updateConfiguration(c2) { _ in }
}
}
}
func stream(_ stream: SCStream, didOutputSampleBuffer sb: CMSampleBuffer, of type: SCStreamOutputType) {
guard type == .screen else { return }
let now = CACurrentMediaTime()
let gapMs = (lastFrameAt == 0) ? 0 : (now - lastFrameAt) * 1000
lastFrameAt = now
frames += 1
if frames <= 10 || frames % 60 == 0 {
print("frames=\(frames) gapMs=\(String(format: "%.1f", gapMs))")
}
}
}