diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 694cefdd..69abf99e 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -89,6 +89,13 @@ + + + { + + private var value: Value + private let lock = NSLock() + + init(wrappedValue value: Value) { + self.value = value + } + + var wrappedValue: Value { + get { load() } + set { store(newValue: newValue) } + } + + func load() -> Value { + lock.lock() + defer { lock.unlock() } + return value + } + + mutating func store(newValue: Value) { + lock.lock() + defer { lock.unlock() } + value = newValue + } +} diff --git a/ios/SolianBroadcastExtension/DarwinNotification.swift b/ios/SolianBroadcastExtension/DarwinNotification.swift new file mode 100644 index 00000000..3501486d --- /dev/null +++ b/ios/SolianBroadcastExtension/DarwinNotification.swift @@ -0,0 +1,29 @@ +// +// DarwinNotificationCenter.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 23/03/2021. +// Copyright © 2021 8x8, Inc. All rights reserved. +// + +import Foundation + +enum DarwinNotification: String { + case broadcastStarted = "iOS_BroadcastStarted" + case broadcastStopped = "iOS_BroadcastStopped" +} + +class DarwinNotificationCenter { + + static let shared = DarwinNotificationCenter() + + private let notificationCenter: CFNotificationCenter + + init() { + notificationCenter = CFNotificationCenterGetDarwinNotifyCenter() + } + + func postNotification(_ name: DarwinNotification) { + CFNotificationCenterPostNotification(notificationCenter, CFNotificationName(rawValue: name.rawValue as CFString), nil, nil, true) + } +} diff --git a/ios/SolianBroadcastExtension/Info.plist b/ios/SolianBroadcastExtension/Info.plist new file mode 100644 index 00000000..e9367904 --- /dev/null +++ b/ios/SolianBroadcastExtension/Info.plist @@ -0,0 +1,15 @@ + + + + + NSExtension + + NSExtensionPointIdentifier + com.apple.broadcast-services-upload + NSExtensionPrincipalClass + $(PRODUCT_MODULE_NAME).SampleHandler + RPBroadcastProcessMode + RPBroadcastProcessModeSampleBuffer + + + diff --git a/ios/SolianBroadcastExtension/SampleHandler.swift b/ios/SolianBroadcastExtension/SampleHandler.swift new file mode 100644 index 00000000..e824b740 --- /dev/null +++ b/ios/SolianBroadcastExtension/SampleHandler.swift @@ -0,0 +1,103 @@ +// +// SampleHandler.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 04.06.2021. +// + +import ReplayKit +import OSLog + +let broadcastLogger = OSLog(subsystem: "dev.solsynth.solian", category: "Broadcast") +private enum Constants { + // the App Group ID value that the app and the broadcast extension targets are setup with. It differs for each app. + static let appGroupIdentifier = "group.solsynth.solian" +} + +class SampleHandler: RPBroadcastSampleHandler { + + private var clientConnection: SocketConnection? + private var uploader: SampleUploader? + + private var frameCount: Int = 0 + + var socketFilePath: String { + let sharedContainer = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: Constants.appGroupIdentifier) + return sharedContainer?.appendingPathComponent("rtc_SSFD").path ?? "" + } + + override init() { + super.init() + if let connection = SocketConnection(filePath: socketFilePath) { + clientConnection = connection + setupConnection() + + uploader = SampleUploader(connection: connection) + } + os_log(.debug, log: broadcastLogger, "%{public}s", socketFilePath) + } + + override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) { + // User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional. + frameCount = 0 + + DarwinNotificationCenter.shared.postNotification(.broadcastStarted) + openConnection() + } + + override func broadcastPaused() { + // User has requested to pause the broadcast. Samples will stop being delivered. + } + + override func broadcastResumed() { + // User has requested to resume the broadcast. Samples delivery will resume. + } + + override func broadcastFinished() { + // User has requested to finish the broadcast. + DarwinNotificationCenter.shared.postNotification(.broadcastStopped) + clientConnection?.close() + } + + override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) { + switch sampleBufferType { + case RPSampleBufferType.video: + uploader?.send(sample: sampleBuffer) + default: + break + } + } +} + +private extension SampleHandler { + + func setupConnection() { + clientConnection?.didClose = { [weak self] error in + os_log(.debug, log: broadcastLogger, "client connection did close \(String(describing: error))") + + if let error = error { + self?.finishBroadcastWithError(error) + } else { + // the displayed failure message is more user friendly when using NSError instead of Error + let JMScreenSharingStopped = 10001 + let customError = NSError(domain: RPRecordingErrorDomain, code: JMScreenSharingStopped, userInfo: [NSLocalizedDescriptionKey: "Screen sharing stopped"]) + self?.finishBroadcastWithError(customError) + } + } + } + + func openConnection() { + let queue = DispatchQueue(label: "broadcast.connectTimer") + let timer = DispatchSource.makeTimerSource(queue: queue) + timer.schedule(deadline: .now(), repeating: .milliseconds(100), leeway: .milliseconds(500)) + timer.setEventHandler { [weak self] in + guard self?.clientConnection?.open() == true else { + return + } + + timer.cancel() + } + + timer.resume() + } +} diff --git a/ios/SolianBroadcastExtension/SampleUploader.swift b/ios/SolianBroadcastExtension/SampleUploader.swift new file mode 100644 index 00000000..6205ad2a --- /dev/null +++ b/ios/SolianBroadcastExtension/SampleUploader.swift @@ -0,0 +1,147 @@ +// +// SampleUploader.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 22/03/2021. +// Copyright © 2021 8x8, Inc. All rights reserved. +// + +import Foundation +import ReplayKit +import OSLog + +private enum Constants { + static let bufferMaxLength = 10240 +} + +class SampleUploader { + + private static var imageContext = CIContext(options: nil) + + @Atomic private var isReady = false + private var connection: SocketConnection + + private var dataToSend: Data? + private var byteIndex = 0 + + private let serialQueue: DispatchQueue + + init(connection: SocketConnection) { + self.connection = connection + self.serialQueue = DispatchQueue(label: "org.jitsi.meet.broadcast.sampleUploader") + + setupConnection() + } + + @discardableResult func send(sample buffer: CMSampleBuffer) -> Bool { + guard isReady else { + return false + } + + isReady = false + + dataToSend = prepare(sample: buffer) + byteIndex = 0 + + serialQueue.async { [weak self] in + self?.sendDataChunk() + } + + return true + } +} + +private extension SampleUploader { + + func setupConnection() { + connection.didOpen = { [weak self] in + self?.isReady = true + } + connection.streamHasSpaceAvailable = { [weak self] in + self?.serialQueue.async { + if let success = self?.sendDataChunk() { + self?.isReady = !success + } + } + } + } + + @discardableResult func sendDataChunk() -> Bool { + guard let dataToSend = dataToSend else { + return false + } + + var bytesLeft = dataToSend.count - byteIndex + var length = bytesLeft > Constants.bufferMaxLength ? Constants.bufferMaxLength : bytesLeft + + length = dataToSend[byteIndex..<(byteIndex + length)].withUnsafeBytes { + guard let ptr = $0.bindMemory(to: UInt8.self).baseAddress else { + return 0 + } + + return connection.writeToStream(buffer: ptr, maxLength: length) + } + + if length > 0 { + byteIndex += length + bytesLeft -= length + + if bytesLeft == 0 { + self.dataToSend = nil + byteIndex = 0 + } + } else { + os_log(.debug, log: broadcastLogger, "writeBufferToStream failure") + } + + return true + } + + func prepare(sample buffer: CMSampleBuffer) -> Data? { + guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else { + os_log(.debug, log: broadcastLogger, "image buffer not available") + return nil + } + + CVPixelBufferLockBaseAddress(imageBuffer, .readOnly) + + let scaleFactor = 1.0 + let width = CVPixelBufferGetWidth(imageBuffer)/Int(scaleFactor) + let height = CVPixelBufferGetHeight(imageBuffer)/Int(scaleFactor) + let orientation = CMGetAttachment(buffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil)?.uintValue ?? 0 + + let scaleTransform = CGAffineTransform(scaleX: CGFloat(1.0/scaleFactor), y: CGFloat(1.0/scaleFactor)) + let bufferData = self.jpegData(from: imageBuffer, scale: scaleTransform) + + CVPixelBufferUnlockBaseAddress(imageBuffer, .readOnly) + + guard let messageData = bufferData else { + os_log(.debug, log: broadcastLogger, "corrupted image buffer") + return nil + } + + let httpResponse = CFHTTPMessageCreateResponse(nil, 200, nil, kCFHTTPVersion1_1).takeRetainedValue() + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Content-Length" as CFString, String(messageData.count) as CFString) + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Width" as CFString, String(width) as CFString) + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Height" as CFString, String(height) as CFString) + CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Orientation" as CFString, String(orientation) as CFString) + + CFHTTPMessageSetBody(httpResponse, messageData as CFData) + + let serializedMessage = CFHTTPMessageCopySerializedMessage(httpResponse)?.takeRetainedValue() as Data? + + return serializedMessage + } + + func jpegData(from buffer: CVPixelBuffer, scale scaleTransform: CGAffineTransform) -> Data? { + let image = CIImage(cvPixelBuffer: buffer).transformed(by: scaleTransform) + + guard let colorSpace = image.colorSpace else { + return nil + } + + let options: [CIImageRepresentationOption: Float] = [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0] + + return SampleUploader.imageContext.jpegRepresentation(of: image, colorSpace: colorSpace, options: options) + } +} diff --git a/ios/SolianBroadcastExtension/SocketConnection.swift b/ios/SolianBroadcastExtension/SocketConnection.swift new file mode 100644 index 00000000..a2a73a9b --- /dev/null +++ b/ios/SolianBroadcastExtension/SocketConnection.swift @@ -0,0 +1,199 @@ +// +// SocketConnection.swift +// Broadcast Extension +// +// Created by Alex-Dan Bumbu on 22/03/2021. +// Copyright © 2021 Atlassian Inc. All rights reserved. +// + +import Foundation +import OSLog + +class SocketConnection: NSObject { + var didOpen: (() -> Void)? + var didClose: ((Error?) -> Void)? + var streamHasSpaceAvailable: (() -> Void)? + + private let filePath: String + private var socketHandle: Int32 = -1 + private var address: sockaddr_un? + + private var inputStream: InputStream? + private var outputStream: OutputStream? + + private var networkQueue: DispatchQueue? + private var shouldKeepRunning = false + + init?(filePath path: String) { + filePath = path + socketHandle = Darwin.socket(AF_UNIX, SOCK_STREAM, 0) + + guard socketHandle != -1 else { + os_log(.debug, log: broadcastLogger, "failure: create socket") + return nil + } + } + + func open() -> Bool { + os_log(.debug, log: broadcastLogger, "open socket connection") + + guard FileManager.default.fileExists(atPath: filePath) else { + os_log(.debug, log: broadcastLogger, "failure: socket file missing") + return false + } + + guard setupAddress() == true else { + return false + } + + guard connectSocket() == true else { + return false + } + + setupStreams() + + inputStream?.open() + outputStream?.open() + + return true + } + + func close() { + unscheduleStreams() + + inputStream?.delegate = nil + outputStream?.delegate = nil + + inputStream?.close() + outputStream?.close() + + inputStream = nil + outputStream = nil + } + + func writeToStream(buffer: UnsafePointer, maxLength length: Int) -> Int { + outputStream?.write(buffer, maxLength: length) ?? 0 + } +} + +extension SocketConnection: StreamDelegate { + + func stream(_ aStream: Stream, handle eventCode: Stream.Event) { + switch eventCode { + case .openCompleted: + os_log(.debug, log: broadcastLogger, "client stream open completed") + if aStream == outputStream { + didOpen?() + } + case .hasBytesAvailable: + if aStream == inputStream { + var buffer: UInt8 = 0 + let numberOfBytesRead = inputStream?.read(&buffer, maxLength: 1) + if numberOfBytesRead == 0 && aStream.streamStatus == .atEnd { + os_log(.debug, log: broadcastLogger, "server socket closed") + close() + notifyDidClose(error: nil) + } + } + case .hasSpaceAvailable: + if aStream == outputStream { + streamHasSpaceAvailable?() + } + case .errorOccurred: + os_log(.debug, log: broadcastLogger, "client stream error occured: \(String(describing: aStream.streamError))") + close() + notifyDidClose(error: aStream.streamError) + + default: + break + } + } +} + +private extension SocketConnection { + + func setupAddress() -> Bool { + var addr = sockaddr_un() + guard filePath.count < MemoryLayout.size(ofValue: addr.sun_path) else { + os_log(.debug, log: broadcastLogger, "failure: fd path is too long") + return false + } + + _ = withUnsafeMutablePointer(to: &addr.sun_path.0) { ptr in + filePath.withCString { + strncpy(ptr, $0, filePath.count) + } + } + + address = addr + return true + } + + func connectSocket() -> Bool { + guard var addr = address else { + return false + } + + let status = withUnsafePointer(to: &addr) { ptr in + ptr.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(socketHandle, $0, socklen_t(MemoryLayout.size)) + } + } + + guard status == noErr else { + os_log(.debug, log: broadcastLogger, "failure: \(status)") + return false + } + + return true + } + + func setupStreams() { + var readStream: Unmanaged? + var writeStream: Unmanaged? + + CFStreamCreatePairWithSocket(kCFAllocatorDefault, socketHandle, &readStream, &writeStream) + + inputStream = readStream?.takeRetainedValue() + inputStream?.delegate = self + inputStream?.setProperty(kCFBooleanTrue, forKey: Stream.PropertyKey(kCFStreamPropertyShouldCloseNativeSocket as String)) + + outputStream = writeStream?.takeRetainedValue() + outputStream?.delegate = self + outputStream?.setProperty(kCFBooleanTrue, forKey: Stream.PropertyKey(kCFStreamPropertyShouldCloseNativeSocket as String)) + + scheduleStreams() + } + + func scheduleStreams() { + shouldKeepRunning = true + + networkQueue = DispatchQueue.global(qos: .userInitiated) + networkQueue?.async { [weak self] in + self?.inputStream?.schedule(in: .current, forMode: .common) + self?.outputStream?.schedule(in: .current, forMode: .common) + RunLoop.current.run() + + var isRunning = false + + repeat { + isRunning = self?.shouldKeepRunning ?? false && RunLoop.current.run(mode: .default, before: .distantFuture) + } while (isRunning) + } + } + + func unscheduleStreams() { + networkQueue?.sync { [weak self] in + self?.inputStream?.remove(from: .current, forMode: .common) + self?.outputStream?.remove(from: .current, forMode: .common) + } + + shouldKeepRunning = false + } + + func notifyDidClose(error: Error?) { + if didClose != nil { + didClose?(error) + } + } +} diff --git a/ios/SolianBroadcastExtension/SolianBroadcastExtension.entitlements b/ios/SolianBroadcastExtension/SolianBroadcastExtension.entitlements new file mode 100644 index 00000000..7121c32b --- /dev/null +++ b/ios/SolianBroadcastExtension/SolianBroadcastExtension.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.solsynth.solian + + + diff --git a/lib/pods/call.dart b/lib/pods/call.dart index c561c654..78c3a51a 100644 --- a/lib/pods/call.dart +++ b/lib/pods/call.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:developer'; import 'package:island/widgets/chat/call_button.dart'; import 'package:livekit_client/livekit_client.dart'; import 'package:freezed_annotation/freezed_annotation.dart'; @@ -203,7 +204,13 @@ class CallNotifier extends _$CallNotifier { Future joinRoom(String roomId) async { if (_roomId == roomId && _room != null) { + log('[Call] Call skipped. Already has data'); return; + } else if (_room != null) { + if (!_room!.isDisposed && + _room!.connectionState != ConnectionState.disconnected) { + throw Exception('Call already connected'); + } } _roomId = roomId; if (_room != null) { @@ -335,5 +342,6 @@ class CallNotifier extends _$CallNotifier { _room?.removeListener(_onRoomChange); _room?.dispose(); _durationTimer?.cancel(); + _roomId = null; } } diff --git a/lib/screens/chat/call.dart b/lib/screens/chat/call.dart index 0f1694f5..ebe33494 100644 --- a/lib/screens/chat/call.dart +++ b/lib/screens/chat/call.dart @@ -1,3 +1,5 @@ +import 'dart:developer'; + import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; import 'package:flutter_hooks/flutter_hooks.dart'; @@ -8,6 +10,7 @@ import 'package:island/widgets/app_scaffold.dart'; import 'package:island/widgets/chat/call_button.dart'; import 'package:island/widgets/chat/call_overlay.dart'; import 'package:island/widgets/chat/call_participant_tile.dart'; +import 'package:island/widgets/alert.dart'; import 'package:livekit_client/livekit_client.dart'; import 'package:material_symbols_icons/symbols.dart'; import 'package:styled_widget/styled_widget.dart'; @@ -23,7 +26,19 @@ class CallScreen extends HookConsumerWidget { final callNotifier = ref.watch(callNotifierProvider.notifier); useEffect(() { - callNotifier.joinRoom(roomId); + log('[Call] Joining the call...'); + callNotifier.joinRoom(roomId).catchError((_) { + showConfirmAlert( + 'Seems there already has a call connected, do you want override it?', + 'Call already connected', + ).then((value) { + if (value != true) return; + log('[Call] Joining the call... with overrides'); + callNotifier.disconnect(); + callNotifier.dispose(); + callNotifier.joinRoom(roomId); + }); + }); return null; }, []); diff --git a/lib/widgets/chat/call_overlay.dart b/lib/widgets/chat/call_overlay.dart index c33dfc2d..26eca838 100644 --- a/lib/widgets/chat/call_overlay.dart +++ b/lib/widgets/chat/call_overlay.dart @@ -63,41 +63,44 @@ class CallControlsBar extends HookConsumerWidget { isScrollControlled: true, useRootNavigator: true, builder: - (context) => ClipRRect( - borderRadius: BorderRadius.only( - topLeft: Radius.circular(8), - topRight: Radius.circular(8), - ), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - ListTile( - leading: const Icon(Symbols.logout, fill: 1), - title: Text('callLeave').tr(), - onTap: () { - callNotifier.disconnect(); - GoRouter.of(context).pop(); - }, - ), - ListTile( - leading: const Icon(Symbols.call_end, fill: 1), - iconColor: Colors.red, - title: Text('callEnd').tr(), - onTap: () async { - callNotifier.disconnect(); - final apiClient = ref.watch(apiClientProvider); + (innerContext) => Column( + mainAxisSize: MainAxisSize.min, + children: [ + ListTile( + leading: const Icon(Symbols.logout, fill: 1), + title: Text('callLeave').tr(), + onTap: () { + callNotifier.disconnect(); + Navigator.of(context).pop(); + Navigator.of(innerContext).pop(); + }, + ), + ListTile( + leading: const Icon(Symbols.call_end, fill: 1), + iconColor: Colors.red, + title: Text('callEnd').tr(), + onTap: () async { + callNotifier.disconnect(); + final apiClient = ref.watch(apiClientProvider); + try { + showLoadingModal(context); await apiClient.delete( '/sphere/chat/realtime/${callNotifier.roomId}', ); callNotifier.dispose(); if (context.mounted) { - GoRouter.of(context).pop(); + Navigator.of(context).pop(); + Navigator.of(innerContext).pop(); } - }, - ), - Gap(MediaQuery.of(context).padding.bottom), - ], - ), + } catch (err) { + showErrorAlert(err); + } finally { + if (context.mounted) hideLoadingModal(context); + } + }, + ), + Gap(MediaQuery.of(context).padding.bottom), + ], ), ), backgroundColor: const Color(0xFFE53E3E), diff --git a/lib/widgets/content/video.web.dart b/lib/widgets/content/video.web.dart index 78cfd1c7..1868c52a 100644 --- a/lib/widgets/content/video.web.dart +++ b/lib/widgets/content/video.web.dart @@ -4,10 +4,12 @@ import 'package:flutter/material.dart'; class UniversalVideo extends StatelessWidget { final String uri; final double aspectRatio; + final bool autoplay; const UniversalVideo({ super.key, required this.uri, required this.aspectRatio, + this.autoplay = false, }); @override diff --git a/pubspec.lock b/pubspec.lock index 1aeeb100..67e1915c 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -573,10 +573,10 @@ packages: dependency: "direct main" description: name: file_picker - sha256: ef9908739bdd9c476353d6adff72e88fd00c625f5b959ae23f7567bd5137db0a + sha256: "13ba4e627ef24503a465d1d61b32596ce10eb6b8903678d362a528f9939b4aa8" url: "https://pub.dev" source: hosted - version: "10.2.0" + version: "10.2.1" file_selector_linux: dependency: transitive description: @@ -1097,10 +1097,10 @@ packages: dependency: "direct main" description: name: google_fonts - sha256: b1ac0fe2832c9cc95e5e88b57d627c5e68c223b9657f4b96e1487aa9098c7b82 + sha256: df9763500dadba0155373e9cb44e202ce21bd9ed5de6bdbd05c5854e86839cb8 url: "https://pub.dev" source: hosted - version: "6.2.1" + version: "6.3.0" graphs: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 186952fe..a7015516 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -53,7 +53,7 @@ dependencies: flutter_highlight: ^0.7.0 uuid: ^4.5.1 url_launcher: ^6.3.2 - google_fonts: ^6.2.1 + google_fonts: ^6.3.0 gap: ^3.0.1 cached_network_image: ^3.4.1 web: ^1.1.1 @@ -73,7 +73,7 @@ dependencies: git: https://github.com/LittleSheep2Code/tus_client.git cross_file: ^0.3.4+2 image_picker: ^1.1.2 - file_picker: ^10.2.0 + file_picker: ^10.2.1 riverpod_annotation: ^2.6.1 image_picker_platform_interface: ^2.10.1 image_picker_android: ^0.8.12+24