From aa94dfcfe09eacd5fbc13c71d9a221166638379e Mon Sep 17 00:00:00 2001 From: LittleSheep Date: Wed, 21 Aug 2024 01:53:16 +0800 Subject: [PATCH] :sparkles: Multipart upload --- lib/models/attachment.dart | 33 +++ lib/providers/attachment_uploader.dart | 188 ++++++++++-------- lib/providers/content/attachment.dart | 100 ++++++++-- lib/screens/account/personalize.dart | 2 +- .../attachments/attachment_editor.dart | 44 ++-- pubspec.lock | 2 +- pubspec.yaml | 1 + 7 files changed, 233 insertions(+), 137 deletions(-) diff --git a/lib/models/attachment.dart b/lib/models/attachment.dart index 5df54b6..bd95afb 100644 --- a/lib/models/attachment.dart +++ b/lib/models/attachment.dart @@ -1,5 +1,30 @@ import 'package:solian/models/account.dart'; +class AttachmentPlaceholder { + int chunkCount; + int chunkSize; + Attachment meta; + + AttachmentPlaceholder({ + required this.chunkCount, + required this.chunkSize, + required this.meta, + }); + + factory AttachmentPlaceholder.fromJson(Map json) => + AttachmentPlaceholder( + chunkCount: json['chunk_count'], + chunkSize: json['chunk_size'], + meta: Attachment.fromJson(json['meta']), + ); + + Map toJson() => { + 'chunk_count': chunkCount, + 'chunk_size': chunkSize, + 'meta': meta.toJson(), + }; +} + class Attachment { int id; DateTime createdAt; @@ -14,7 +39,9 @@ class Attachment { String hash; int destination; bool isAnalyzed; + bool isUploaded; Map? metadata; + Map? fileChunks; bool isMature; Account? account; int? accountId; @@ -33,7 +60,9 @@ class Attachment { required this.hash, required this.destination, required this.isAnalyzed, + required this.isUploaded, required this.metadata, + required this.fileChunks, required this.isMature, required this.account, required this.accountId, @@ -55,7 +84,9 @@ class Attachment { hash: json['hash'], destination: json['destination'], isAnalyzed: json['is_analyzed'], + isUploaded: json['is_uploaded'], metadata: json['metadata'], + fileChunks: json['file_chunks'], isMature: json['is_mature'], account: json['account'] != null ? Account.fromJson(json['account']) : null, @@ -76,7 +107,9 @@ class Attachment { 'hash': hash, 'destination': destination, 'is_analyzed': isAnalyzed, + 'is_uploaded': isUploaded, 'metadata': metadata, + 'file_chunks': fileChunks, 'is_mature': isMature, 'account': account?.toJson(), 'account_id': accountId, diff --git a/lib/providers/attachment_uploader.dart b/lib/providers/attachment_uploader.dart index e8f643a..9945ee9 100644 --- a/lib/providers/attachment_uploader.dart +++ b/lib/providers/attachment_uploader.dart @@ -1,15 +1,17 @@ import 'dart:async'; -import 'dart:io'; import 'dart:typed_data'; +import 'package:cross_file/cross_file.dart'; import 'package:get/get.dart'; +import 'package:path/path.dart' show basename; import 'package:solian/models/attachment.dart'; import 'package:solian/providers/content/attachment.dart'; class AttachmentUploadTask { - File file; - String usage; + XFile file; + String pool; Map? metadata; + Map? chunkFiles; double progress = 0; bool isUploading = false; @@ -18,7 +20,7 @@ class AttachmentUploadTask { AttachmentUploadTask({ required this.file, - required this.usage, + required this.pool, this.metadata, }); } @@ -75,30 +77,33 @@ class AttachmentUploaderController extends GetxController { queueOfUpload[queueIndex].isUploading = true; final task = queueOfUpload[queueIndex]; - final result = await _rawUploadAttachment( - await task.file.readAsBytes(), - task.file.path, - task.usage, - null, - onProgress: (value) { - queueOfUpload[queueIndex].progress = value; - _progressOfUpload = value; - }, - onError: (err) { - queueOfUpload[queueIndex].error = err; - queueOfUpload[queueIndex].isUploading = false; - }, - ); + try { + final result = await _chunkedUploadAttachment( + task.file, + task.pool, + null, + onData: (_) {}, + onProgress: (progress) { + queueOfUpload[queueIndex].progress = progress; + _progressOfUpload = progress; + }, + ); + return result; + } catch (err) { + queueOfUpload[queueIndex].error = err; + queueOfUpload[queueIndex].isUploading = false; + } finally { + _progressOfUpload = 1; + if (queueOfUpload[queueIndex].error == null) { + queueOfUpload.removeAt(queueIndex); + } + _stopProgressSyncTimer(); + _syncProgress(); - if (queueOfUpload[queueIndex].error == null) { - queueOfUpload.removeAt(queueIndex); + isUploading.value = false; } - _stopProgressSyncTimer(); - _syncProgress(); - isUploading.value = false; - - return result; + return null; } Future performUploadQueue({ @@ -117,22 +122,23 @@ class AttachmentUploaderController extends GetxController { queueOfUpload[idx].isUploading = true; final task = queueOfUpload[idx]; - final result = await _rawUploadAttachment( - await task.file.readAsBytes(), - task.file.path, - task.usage, - null, - onProgress: (value) { - queueOfUpload[idx].progress = value; - _progressOfUpload = (idx + value) / queueOfUpload.length; - }, - onError: (err) { - queueOfUpload[idx].error = err; - queueOfUpload[idx].isUploading = false; - }, - ); - _progressOfUpload = (idx + 1) / queueOfUpload.length; - if (result != null) onData(result); + try { + final result = await _chunkedUploadAttachment( + task.file, + task.pool, + null, + onData: (_) {}, + onProgress: (progress) { + queueOfUpload[idx].progress = progress; + }, + ); + if (result != null) onData(result); + } catch (err) { + queueOfUpload[idx].error = err; + queueOfUpload[idx].isUploading = false; + } finally { + _progressOfUpload = (idx + 1) / queueOfUpload.length; + } queueOfUpload[idx].isUploading = false; queueOfUpload[idx].isCompleted = true; @@ -145,69 +151,75 @@ class AttachmentUploaderController extends GetxController { isUploading.value = false; } - Future uploadAttachmentWithCallback( - Uint8List data, - String path, - String pool, - Map? metadata, - Function(Attachment?) callback, - ) async { - if (isUploading.value) throw Exception('uploading blocked'); - - isUploading.value = true; - final result = await _rawUploadAttachment( - data, - path, - pool, - metadata, - onProgress: (progress) { - progressOfUpload.value = progress; - }, - ); - isUploading.value = false; - callback(result); - } - - Future uploadAttachment( + Future uploadAttachmentFromData( Uint8List data, String path, String pool, Map? metadata, ) async { if (isUploading.value) throw Exception('uploading blocked'); - isUploading.value = true; - final result = await _rawUploadAttachment( - data, - path, - pool, - metadata, - onProgress: (progress) { - progressOfUpload.value = progress; - }, - ); - isUploading.value = false; - return result; - } - Future _rawUploadAttachment( - Uint8List data, String path, String pool, Map? metadata, - {Function(double)? onProgress, Function(dynamic err)? onError}) async { - final AttachmentProvider provider = Get.find(); + final AttachmentProvider attach = Get.find(); + try { - final result = await provider.createAttachment( + final result = await attach.createAttachmentDirectly( data, path, pool, metadata, - onProgress: onProgress, ); return result; - } catch (err) { - if (onError != null) { - onError(err); - } + } catch (_) { return null; + } finally { + isUploading.value = false; } } + + Future _chunkedUploadAttachment( + XFile file, + String pool, + Map? metadata, { + required Function(AttachmentPlaceholder) onData, + required Function(double) onProgress, + }) async { + final AttachmentProvider attach = Get.find(); + + final holder = await attach.createAttachmentMultipartPlaceholder( + await file.length(), + file.path, + pool, + metadata, + ); + onData(holder); + + onProgress(0); + + final filename = basename(file.path); + final chunks = holder.meta.fileChunks ?? {}; + var currentTask = 0; + for (final entry in chunks.entries) { + final beginCursor = entry.value * holder.chunkSize; + final endCursor = (entry.value + 1) * holder.chunkSize; + final data = Uint8List.fromList(await file + .openRead(beginCursor, endCursor) + .expand((chunk) => chunk) + .toList()); + + final out = await attach.uploadAttachmentMultipartChunk( + data, + filename, + holder.meta.rid, + entry.key, + ); + holder.meta = out; + + currentTask++; + onProgress(currentTask / chunks.length); + onData(holder); + } + + return holder.meta; + } } diff --git a/lib/providers/content/attachment.dart b/lib/providers/content/attachment.dart index 81ff5af..20459c4 100644 --- a/lib/providers/content/attachment.dart +++ b/lib/providers/content/attachment.dart @@ -7,7 +7,6 @@ import 'package:solian/models/attachment.dart'; import 'package:solian/models/pagination.dart'; import 'package:solian/providers/auth.dart'; import 'package:solian/services.dart'; -import 'package:dio/dio.dart' as dio; class AttachmentProvider extends GetConnect { static Map mimetypeOverrides = { @@ -83,16 +82,21 @@ class AttachmentProvider extends GetConnect { return null; } - Future createAttachment( - Uint8List data, String path, String pool, Map? metadata, - {Function(double)? onProgress}) async { + Future createAttachmentDirectly( + Uint8List data, + String path, + String pool, + Map? metadata, + ) async { final AuthProvider auth = Get.find(); if (auth.isAuthorized.isFalse) throw Exception('unauthorized'); - await auth.ensureCredentials(); + final client = auth.configureClient( + 'uc', + timeout: const Duration(minutes: 3), + ); - final filePayload = - dio.MultipartFile.fromBytes(data, filename: basename(path)); + final filePayload = MultipartFile(data, filename: basename(path)); final fileAlt = basename(path).contains('.') ? basename(path).substring(0, basename(path).lastIndexOf('.')) : basename(path); @@ -105,30 +109,82 @@ class AttachmentProvider extends GetConnect { if (mimetypeOverrides.keys.contains(fileExt)) { mimetypeOverride = mimetypeOverrides[fileExt]; } - final payload = dio.FormData.fromMap({ + final payload = FormData({ 'alt': fileAlt, 'file': filePayload, 'pool': pool, if (mimetypeOverride != null) 'mimetype': mimetypeOverride, 'metadata': jsonEncode(metadata), }); - final resp = await dio.Dio( - dio.BaseOptions( - baseUrl: ServiceFinder.buildUrl('files', null), - headers: {'Authorization': 'Bearer ${auth.credentials!.accessToken}'}, - ), - ).post( - '/attachments', - data: payload, - onSendProgress: (count, total) { - if (onProgress != null) onProgress(count / total); - }, - ); + final resp = await client.post('/attachments', payload); if (resp.statusCode != 200) { - throw Exception(resp.data); + throw Exception(resp.bodyString); } - return Attachment.fromJson(resp.data); + return Attachment.fromJson(resp.body); + } + + Future createAttachmentMultipartPlaceholder( + int size, + String path, + String pool, + Map? metadata, + ) async { + final AuthProvider auth = Get.find(); + if (auth.isAuthorized.isFalse) throw Exception('unauthorized'); + + final client = auth.configureClient('uc'); + + final fileAlt = basename(path).contains('.') + ? basename(path).substring(0, basename(path).lastIndexOf('.')) + : basename(path); + final fileExt = basename(path) + .substring(basename(path).lastIndexOf('.') + 1) + .toLowerCase(); + + // Override for some files cannot be detected mimetype by server-side + String? mimetypeOverride; + if (mimetypeOverrides.keys.contains(fileExt)) { + mimetypeOverride = mimetypeOverrides[fileExt]; + } + final resp = await client.post('/attachments/multipart', { + 'alt': fileAlt, + 'name': basename(path), + 'size': size, + 'pool': pool, + if (mimetypeOverride != null) 'mimetype': mimetypeOverride, + 'metadata': metadata, + }); + if (resp.statusCode != 200) { + throw Exception(resp.bodyString); + } + + return AttachmentPlaceholder.fromJson(resp.body); + } + + Future uploadAttachmentMultipartChunk( + Uint8List data, + String name, + String rid, + String cid, + ) async { + final AuthProvider auth = Get.find(); + if (auth.isAuthorized.isFalse) throw Exception('unauthorized'); + + final client = auth.configureClient( + 'uc', + timeout: const Duration(minutes: 3), + ); + + final payload = FormData({ + 'file': MultipartFile(data, filename: name), + }); + final resp = await client.post('/attachments/multipart/$rid/$cid', payload); + if (resp.statusCode != 200) { + throw Exception(resp.bodyString); + } + + return Attachment.fromJson(resp.body); } Future updateAttachment( diff --git a/lib/screens/account/personalize.dart b/lib/screens/account/personalize.dart index 34418ad..ec02c92 100644 --- a/lib/screens/account/personalize.dart +++ b/lib/screens/account/personalize.dart @@ -113,7 +113,7 @@ class _PersonalizeScreenState extends State { Attachment? attachResult; try { - attachResult = await provider.createAttachment( + attachResult = await provider.createAttachmentDirectly( await file.readAsBytes(), file.path, 'avatar', diff --git a/lib/widgets/attachments/attachment_editor.dart b/lib/widgets/attachments/attachment_editor.dart index 098a1df..15f859e 100644 --- a/lib/widgets/attachments/attachment_editor.dart +++ b/lib/widgets/attachments/attachment_editor.dart @@ -72,8 +72,8 @@ class _AttachmentEditorPopupState extends State { if (medias.isEmpty) return; _enqueueTaskBatch(medias.map((x) { - final file = File(x.path); - return AttachmentUploadTask(file: file, usage: widget.pool); + final file = XFile(x.path); + return AttachmentUploadTask(file: file, pool: widget.pool); })); } else { final media = await _imagePicker.pickMedia( @@ -83,7 +83,7 @@ class _AttachmentEditorPopupState extends State { if (media == null) return; _enqueueTask( - AttachmentUploadTask(file: File(media.path), usage: widget.pool), + AttachmentUploadTask(file: XFile(media.path), pool: widget.pool), ); } } @@ -95,9 +95,8 @@ class _AttachmentEditorPopupState extends State { final media = await _imagePicker.pickVideo(source: ImageSource.gallery); if (media == null) return; - final file = File(media.path); _enqueueTask( - AttachmentUploadTask(file: file, usage: widget.pool), + AttachmentUploadTask(file: XFile(media.path), pool: widget.pool), ); } @@ -113,7 +112,7 @@ class _AttachmentEditorPopupState extends State { List files = result.paths.map((path) => File(path!)).toList(); _enqueueTaskBatch(files.map((x) { - return AttachmentUploadTask(file: x, usage: widget.pool); + return AttachmentUploadTask(file: XFile(x.path), pool: widget.pool); })); } @@ -129,9 +128,8 @@ class _AttachmentEditorPopupState extends State { } if (media == null) return; - final file = File(media.path); _enqueueTask( - AttachmentUploadTask(file: file, usage: widget.pool), + AttachmentUploadTask(file: XFile(media.path), pool: widget.pool), ); } @@ -197,20 +195,16 @@ class _AttachmentEditorPopupState extends State { if (_uploadController.isUploading.value) return; - _uploadController.uploadAttachmentWithCallback( - data, - 'Pasted Image', - widget.pool, - null, - (item) { - if (item == null) return; - widget.onAdd(item.rid); - if (mounted) { - setState(() => _attachments.add(item)); - if (widget.singleMode) Navigator.pop(context); - } - }, - ); + _uploadController + .uploadAttachmentFromData(data, 'Pasted Image', widget.pool, null) + .then((item) { + if (item == null) return; + widget.onAdd(item.rid); + if (mounted) { + setState(() => _attachments.add(item)); + if (widget.singleMode) Navigator.pop(context); + } + }); } String _formatBytes(int bytes, {int decimals = 2}) { @@ -304,7 +298,7 @@ class _AttachmentEditorPopupState extends State { ], ); if (croppedFile == null) return; - _uploadController.queueOfUpload[queueIndex].file = File(croppedFile.path); + _uploadController.queueOfUpload[queueIndex].file = XFile(croppedFile.path); _uploadController.queueOfUpload.refresh(); } @@ -581,8 +575,8 @@ class _AttachmentEditorPopupState extends State { onDragDone: (detail) async { if (_uploadController.isUploading.value) return; _enqueueTaskBatch(detail.files.map((x) { - final file = File(x.path); - return AttachmentUploadTask(file: file, usage: widget.pool); + final file = XFile(x.path); + return AttachmentUploadTask(file: file, pool: widget.pool); })); }, child: Column( diff --git a/pubspec.lock b/pubspec.lock index f09fb4a..2b0aec9 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -263,7 +263,7 @@ packages: source: hosted version: "3.1.1" cross_file: - dependency: transitive + dependency: "direct main" description: name: cross_file sha256: "7caf6a750a0c04effbb52a676dce9a4a592e10ad35c34d6d2d0e4811160d5670" diff --git a/pubspec.yaml b/pubspec.yaml index d4b1999..82fa37c 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -72,6 +72,7 @@ dependencies: media_kit_video: ^1.2.4 media_kit_libs_video: ^1.0.4 flutter_svg: ^2.0.10+1 + cross_file: ^0.3.4+2 dev_dependencies: flutter_test: