Uploading chunked video file to cloudinary in flutter - flutter

I am trying to upload a video file thats greater than 100mb to cloudinary in chunks.
I have created a class that splits the video file and uploads each chunk.
import 'dart:async';
import 'dart:io';
import 'dart:math';
import 'package:http/http.dart' as http;
import 'network.dart';
class ChunkedUploader {
final http.Client _client;
ChunkedUploader(this._client);
Future<http.StreamedResponse> upload({
String url,
File file,
Map<String, dynamic> body,
int maxChunkSize,
int uniqueId,
Function(double) onUploadProgress,
}) =>
UploadRequest(
_client,
url: url,
file: file,
body: body,
maxChunkSize: maxChunkSize,
uniqueId: uniqueId,
onUploadProgress: onUploadProgress,
).upload();
}
class UploadRequest {
final http.Client client;
final String url;
final File file;
final Map<String, dynamic> body;
final int uniqueId;
final Function(double) onUploadProgress;
int _fileSize;
int _maxChunkSize;
UploadRequest(
this.client, {
this.url,
this.file,
this.body,
this.uniqueId,
this.onUploadProgress,
int maxChunkSize,
}) {
_fileSize = file.lengthSync();
_maxChunkSize = min(_fileSize, maxChunkSize ?? _fileSize);
}
Future<http.StreamedResponse> upload() async {
http.StreamedResponse finalResponse;
for (int i = 0; i < _chunksCount; i++) {
final start = _getChunkStart(i);
final end = _getChunkEnd(i);
final chunkStream = _getChunkStream(start, end);
MultipartRequest request = MultipartRequest(
"POST",
Uri.parse(url),
onProgress: (int bytes, int total) {
_updateProgress(i, bytes, total);
},
);
//Add the headers
request.headers.addAll(_getHeaders(start, end));
//Add the body
request.fields.addAll(body);
//Get the file
http.MultipartFile multipartFile =
http.MultipartFile("file", chunkStream, end - start);
request.files.add(multipartFile);
//Send the file
http.StreamedResponse response =
await client.send(request).catchError((error) {
throw error;
});
print(response);
finalResponse = response;
}
return finalResponse;
}
Stream<List<int>> _getChunkStream(int start, int end) =>
file.openRead(start, end);
// Updating total upload progress
_updateProgress(int chunkIndex, int chunkCurrent, int chunkTotal) {
int totalUploadedSize = (chunkIndex * _maxChunkSize) + chunkCurrent;
double totalUploadProgress = (totalUploadedSize / _fileSize) * 100;
this.onUploadProgress?.call(totalUploadProgress);
}
// Returning start byte offset of current chunk
int _getChunkStart(int chunkIndex) => chunkIndex * _maxChunkSize;
// Returning end byte offset of current chunk
int _getChunkEnd(int chunkIndex) =>
min((chunkIndex + 1) * _maxChunkSize, _fileSize);
// Returning a header map object containing Content-Range
// https://tools.ietf.org/html/rfc7233#section-2
Map<String, String> _getHeaders(int start, int end) => {
'X-Unique-Upload-Id': "$uniqueId",
'Content-Range': 'bytes $start-${end - 1}/$_fileSize',
};
// Returning chunks count based on file size and maximum chunk size
int get _chunksCount => (_fileSize / _maxChunkSize).ceil();
}
When making the request i am getting a response of
Status code - 500
Body - {
“error” : {
“message” : "General Error”
}
}
I have been in touch with cloudinary and they have found the request and said that it is failing with the error: invalid byte sequence in UTF-8.
Any help would be appreciated

Related

Flutter Application Freezing when download large file

Now I made a script that encrypts my videos and downloads them to the application storage, But now when I try to download a small file size I don't face any issues, but when I try to download a large file my application is stuck at 99% and take about 2 minutes to save the file to application storage.
`[![99% of file downloading](https://i.stack.imgur.com/W3UUu.jpg)](https://i.stack.imgur.com/W3UUu.jpg)
My Code
`
final String url = streamInfo.url.toString();
final dir = await getApplicationDocumentsDirectory();
String appDocPath = dir.path;
print("Downloading...");
var resp = await dio.get(url,
options: Options(
responseType: ResponseType.bytes,
followRedirects: false,
),
onReceiveProgress: (recivedBytes, totalBytes) {
setState(() {
progress = recivedBytes / totalBytes;
});
},
);
print(resp.data);
var encResult = _encryptData(resp.data);
_writeData(encResult, appDocPath + '/${widget.lessoneName.toString()}.aes');
print("File downloaded successfully");`
```
```
_encryptData(str){
final encrypted = MyEncrypt.myEncrypt.encryptBytes(str,iv:MyEncrypt.myIv);
return encrypted.bytes;
}
Future<String> _writeData(str,path) async{
print("Writting data");
File f = File(path);
print(f);
await f.writeAsBytes(str);
return f.absolute.toString();
}
````
```
class MyEncrypt{
static final myKey = esc.Key.fromUtf8('TechWithVPTechWithVPTechWithVP12');
static final myIv = esc.IV.fromUtf8('VivekPanacha1122');
static final myEncrypt = esc.Encrypter(esc.AES(myKey));
}
```
maybe you have found a solution to this issue
anyway I will write the correct solution for others who are facing such issues
you have to use isolate to reading data as bytes so that it has not affected on UI
with isolate, you send expensive operations to the background which means you made a new thread so that you catch the result after it done
by the way, remember your function (in this case read data as bytes) should be high-level method
here is the sample code
import 'dart:io';
import 'dart:isolate';
import 'package:dio/dio.dart' as dio;
import 'package:encrypt/encrypt.dart';
import 'package:path_provider/path_provider.dart';
class DownloadFileModel {
final SendPort sendPort;
final dio.Response<dynamic> response;
final String savePath;
DownloadVideoModel({
required this.response,
required this.sendPort,
required this.savePath,
});
}
class DownloadFile {
dio.Dio request = dio.Dio();
void downloadNewFile(String url) async {
final dir = await getApplicationDocumentsDirectory();
String appDocPath = dir.path;
var resp = await request.get(
url,
options: dio.Options(
responseType: dio.ResponseType.bytes,
followRedirects: false,
),
onReceiveProgress: (receivedBytes, totalBytes) {
print(receivedBytes / totalBytes);
},
);
ReceivePort port = ReceivePort();
Isolate.spawn(
whenDownloadCompleted,
DownloadFileModel(
response: resp, sendPort: port.sendPort, savePath: appDocPath),
);
port.listen((encryptedFilePath) {
print(encryptedFilePath);
port.close();
});
}
}
class MyEncrypt {
static final myKey = Key.fromUtf8('TechWithVPTechWithVPTechWithVP12');
static final myIv = IV.fromUtf8('VivekPanacha1122');
static final myEncrypt = Encrypter(AES(myKey));
}
void whenDownloadCompleted(DownloadVideoModel model) async {
SendPort sendPort = model.sendPort;
var encryptResult =
MyEncrypt.myEncrypt.encryptBytes(iv: MyEncrypt.myIv, model.response.data);
File encryptedFile = File("${model.savePath}/myFile.aes");
encryptedFile.writeAsBytes(encryptResult.bytes);
sendPort.send(encryptedFile.absolute.path);
}
For more info head over to flutter official document site
https://api.flutter.dev/flutter/dart-isolate/Isolate-class.html

Parse data Map from json fail, how to fix

I've method search from api, data model like object, but i don't know how to parse, anyone can help me let see my code below, i have search method and api link, if u wanna other code, u can comment on column comment
import 'package:get/get.dart';
import 'package:resto_app/app/models/detail_model.dart';
import 'package:resto_app/shared/constant/api_services.dart';
import 'package:resto_app/shared/services/base_client.dart';
class SearchController extends GetxController {
//TODO: Implement SearchController
RxBool isAwaitData = false.obs;
// RxList<DetailModel> ;
Future<List<DetailModel>> searchController(
{String query = '', bool isWait = true}) async {
if (isWait) {
isAwaitData.value = true;
List result = await BaseClient().get(
baseUrl: ApiService.baseURL, api: ApiService.searchResto + query);
// var response = await http
// .get(Uri.parse(ApiService.baseURL + ApiService.searchResto + query));
return result.map((e) => DetailModel.fromJson(e)).where((resto) {
final nameLower = resto.restaurant.name.toLowerCase();
final cityLower = resto.restaurant.city.toLowerCase();
final searchLower = query.toLowerCase();
return nameLower.contains(cityLower);
}).toList();
} else {
throw Exception();
}
// if (response.statusCode == 200) {
// final result = Map<String, dynamic>.from(jsonDecode(response.body));
// result.map((key, value) =>
// key.contains(result)
// );
// }
}
}
class ApiService {
static const baseURL = 'https://restaurant-api.dicoding.dev';
static const allResto = '/list';
static const detailPage = '/detail/';
static const searchResto = '/search?q=';
}
The API doesn't return a list, but a map - The restaurants key of the response points to a list of objects.
Try:
final Map<String, dynamic> responseData = await BaseClient().get(
baseUrl: ApiService.baseURL,
api: ApiService.searchResto + query
);
final restaurantData = responseData['restaurants'];
return restaurantData.map((e) => DetailModel.fromJson(e)).where((resto) {
final nameLower = resto.restaurant.name.toLowerCase();
final cityLower = resto.restaurant.city.toLowerCase();
final searchLower = query.toLowerCase();
return nameLower.contains(cityLower);
}).toList();

how create custom download in flutter

How to create a custom download with pause and resume, pause in flutter using http or dio I think this code is good but lacks pause and resume
import 'dart:typed_data';
import 'dart:io';
import 'package:http/http.dart';
import 'package:path_provider/path_provider.dart';
downloadFile(String url, {String filename}) async {
var httpClient = http.Client();
var request = new http.Request('GET', Uri.parse(url));
var response = httpClient.send(request);
String dir = (await getApplicationDocumentsDirectory()).path;
response.asStream().listen((http.StreamedResponse r) {
List<List<int>> chunks = new List();
int downloaded = 0;
r.stream.listen((List<int> chunk) {
// Display percentage of completion
debugPrint('downloadPercentage: ${downloaded / r.contentLength * 100}');
chunks.add(chunk);
downloaded += chunk.length;
}, onDone: () async {
// Display percentage of completion
debugPrint('downloadPercentage: ${downloaded / r.contentLength * 100}');
// Save the file
File file = new File('$dir/$filename');
final Uint8List bytes = Uint8List(r.ntentLength);
int offset = 0;
for (List<int> chunk in chunks) {
bytes.setRange(offset, offset + chunk.length, chunk);
offset += chunk.length;
}
await file.writeAsBytes(bytes);
return;
});
}
you can try this package https://pub.dev/packages/flutter_downloader
FlutterDownloader.pause(taskId: taskId);
FlutterDownloader.resume(taskId: taskId);

Flutter web: How to Upload a Large File?

Is there a way to upload large files to server?
I am using MultipartRequest with MultipartFile like:
List<int> fileBytes) async {
var request = new http.MultipartRequest("POST", Uri.parse(url));
request.files.add(http.MultipartFile.fromBytes(
'file',
fileBytes,
contentType: MediaType('application', 'octet-stream'),
filename: fileName));
request.headers.addAll(headers);
var streamedResponse = await request.send();
return await http.Response.fromStream(streamedResponse);
and reading the file like:
html.InputElement uploadInput = html.FileUploadInputElement();
uploadInput.multiple = false;
uploadInput.draggable = true;
uploadInput.click();
uploadInput.onChange.listen((e) {
final files = uploadInput.files;
final file = files[0];
final reader = new html.FileReader();
reader.onLoadEnd.listen((e) {
setState(() {
_bytesData =
Base64Decoder().convert(reader.result.toString().split(",").last);
_selectedFile = _bytesData;
});
});
reader.readAsDataUrl(file);
});
It is OK for files around 30 MB but for more than that, I am getting Error code: Out of Memory.
Am I doing something wrong? I saw somewhere
MultipartFile.fromBytes will give you some issues on bigger files, as the browser will limit your memory consumption.
And I think his solution is:
There’s a fromStream constructor. Usually, for bigger files, I just use HttpRequest, and put the File object in a FormData instance.
I used MultipartFile and MultipartFile.fromString and both times (for 150 MB file) that happened again.
How can I use this solution? or Is there a better way to do that for files more than 500 MB?
Update
Added an answer using Worker. This is not a great solution but I think this might help someone.
Currently, I solved the problem using this approach:
Import:
import 'package:universal_html/html.dart' as html;
Flutter part:
class Upload extends StatefulWidget {
#override
_UploadState createState() => _UploadState();
}
class _UploadState extends State<Upload> {
html.Worker myWorker;
html.File file;
_uploadFile() async {
String _uri = "/upload";
myWorker.postMessage({"file": file, "uri": _uri});
}
_selectFile() {
html.InputElement uploadInput = html.FileUploadInputElement();
uploadInput.multiple = false;
uploadInput.click();
uploadInput.onChange.listen((e) {
file = uploadInput.files.first;
});
}
#override
void initState() {
myWorker = new html.Worker('upload_worker.js');
myWorker.onMessage.listen((e) {
setState(() {
//progressbar,...
});
});
super.initState();
}
#override
Widget build(BuildContext context) {
return Column(
children: [
RaisedButton(
onPressed: _selectFile(),
child: Text("Select File"),
),
RaisedButton(
onPressed: _uploadFile(),
child: Text("Upload"),
),
],
);
}
}
Javascript part:
In the web folder (next to index.html), create the file 'upload_worker.js' .
self.addEventListener('message', async (event) => {
var file = event.data.file;
var url = event.data.uri;
uploadFile(file, url);
});
function uploadFile(file, url) {
var xhr = new XMLHttpRequest();
var formdata = new FormData();
var uploadPercent;
formdata.append('file', file);
xhr.upload.addEventListener('progress', function (e) {
//Use this if you want to have a progress bar
if (e.lengthComputable) {
uploadPercent = Math.floor((e.loaded / e.total) * 100);
postMessage(uploadPercent);
}
}, false);
xhr.onreadystatechange = function () {
if (xhr.readyState == XMLHttpRequest.DONE) {
postMessage("done");
}
}
xhr.onerror = function () {
// only triggers if the request couldn't be made at all
postMessage("Request failed");
};
xhr.open('POST', url, true);
xhr.send(formdata);
}
I solved the problem using only Dart code: The way to go is to use a chunk uploader.
This means to manually send the file in little parts. I send 99MB per request for example.
There is already a basic implementation of this online:
https://pub.dev/packages/chunked_uploader
You have to get a stream, this is possible with the file_picker or the drop_zone library. I used the drop_zone library because it provides the file picker and the drop zone functionality. In my code the dynamic file objects come from the drop_zone library.
Maybe you have to adjust the chunk uploader functionality depending one your backend. I use a django backend where I wrote a simple view that saves the files. In case of small files it can receive multipart requests with multiple files, in case of large files it can receive chunks and continiues to write a file if a previous chunk was received.
Here some parts of my code:
Python backend:
#api_view(["POST"])
def upload(request):
basePath = config.get("BasePath")
targetFolder = os.path.join(basePath, request.data["taskId"], "input")
if not os.path.exists(targetFolder):
os.makedirs(targetFolder)
for count, file in enumerate(request.FILES.getlist("Your parameter name on server side")):
path = os.path.join(targetFolder, file.name)
print(path)
with open(path, 'ab') as destination:
for chunk in file.chunks():
destination.write(chunk)
return HttpResponse("File(s) uploaded!")
flutter chunk uploader in my version:
import 'dart:async';
import 'dart:html';
import 'dart:math';
import 'package:dio/dio.dart';
import 'package:flutter_dropzone/flutter_dropzone.dart';
import 'package:http/http.dart' as http;
class UploadRequest {
final Dio dio;
final String url;
final String method;
final String fileKey;
final Map<String, String>? bodyData;
final Map<String, String>? headers;
final CancelToken? cancelToken;
final dynamic file;
final Function(double)? onUploadProgress;
late final int _maxChunkSize;
int fileSize;
String fileName;
late DropzoneViewController controller;
UploadRequest(
this.dio, {
required this.url,
this.method = "POST",
this.fileKey = "file",
this.bodyData = const {},
this.cancelToken,
required this.file,
this.onUploadProgress,
int maxChunkSize = 1024 * 1024 * 99,
required this.controller,
required this.fileSize,
required this.fileName,
this.headers
}) {
_maxChunkSize = min(fileSize, maxChunkSize);
}
Future<Response?> upload() async {
Response? finalResponse;
for (int i = 0; i < _chunksCount; i++) {
final start = _getChunkStart(i);
print("start is $start");
final end = _getChunkEnd(i);
final chunkStream = _getChunkStream(start, end);
var request = http.MultipartRequest(
"POST",
Uri.parse(url),
);
//request.headers.addAll(_getHeaders(start, end));
request.headers.addAll(headers!);
//-----add other fields if needed
request.fields.addAll(bodyData!);
request.files.add(http.MultipartFile(
"Your parameter name on server side",
chunkStream,
fileSize,
filename: fileName// + i.toString(),
)
);
//-------Send request
var resp = await request.send();
//------Read response
String result = await resp.stream.bytesToString();
//-------Your response
print(result);
}
return finalResponse;
}
Stream<List<int>> _getChunkStream(int start, int end) async* {
print("reading from $start to $end");
final reader = FileReader();
final blob = file.slice(start, end);
reader.readAsArrayBuffer(blob);
await reader.onLoad.first;
yield reader.result as List<int>;
}
// Updating total upload progress
_updateProgress(int chunkIndex, int chunkCurrent, int chunkTotal) {
int totalUploadedSize = (chunkIndex * _maxChunkSize) + chunkCurrent;
double totalUploadProgress = totalUploadedSize / fileSize;
this.onUploadProgress?.call(totalUploadProgress);
}
// Returning start byte offset of current chunk
int _getChunkStart(int chunkIndex) => chunkIndex * _maxChunkSize;
// Returning end byte offset of current chunk
int _getChunkEnd(int chunkIndex) =>
min((chunkIndex + 1) * _maxChunkSize, fileSize);
// Returning a header map object containing Content-Range
// https://tools.ietf.org/html/rfc7233#section-2
Map<String, String> _getHeaders(int start, int end) {
var header = {'Content-Range': 'bytes $start-${end - 1}/$fileSize'};
if (headers != null) {
header.addAll(headers!);
}
return header;
}
// Returning chunks count based on file size and maximum chunk size
int get _chunksCount {
var result = (fileSize / _maxChunkSize).ceil();
return result;
}
}
Upload code that decides whether to upload multiple files in one request or one file divided to many requests:
//upload the large files
Map<String, String> headers = {
'Authorization': requester.loginToken!
};
fileUploadView.droppedFiles.sort((a, b) => b.size - a.size);
//calculate the sum of teh files:
double sumInMb = 0;
int divideBy = 1000000;
for (UploadableFile file in fileUploadView.droppedFiles) {
sumInMb += file.size / divideBy;
}
var dio = Dio();
int uploadedAlready = 0;
for (UploadableFile file in fileUploadView.droppedFiles) {
if (sumInMb < 99) {
break;
}
var uploadRequest = UploadRequest(
dio,
url: requester.backendApi+ "/upload",
file: file.file,
controller: fileUploadView.controller!,
fileSize: file.size,
fileName: file.name,
headers: headers,
bodyData: {
"taskId": taskId.toString(),
"user": requester.username!,
},
);
await uploadRequest.upload();
uploadedAlready++;
sumInMb -= file.size / divideBy;
}
if (uploadedAlready > 0) {
fileUploadView.droppedFiles.removeRange(0, uploadedAlready);
}
print("large files uploaded");
// upload the small files
//---Create http package multipart request object
var request = http.MultipartRequest(
"POST",
Uri.parse(requester.backendApi+ "/upload"),
);
request.headers.addAll(headers);
//-----add other fields if needed
request.fields["taskId"] = taskId.toString();
print("adding files selected with drop zone");
for (UploadableFile file in fileUploadView.droppedFiles) {
Stream<List<int>>? stream = fileUploadView.controller?.getFileStream(file.file);
print("sending " + file.name);
request.files.add(http.MultipartFile(
"Your parameter name on server side",
stream!,
file.size,
filename: file.name));
}
//-------Send request
var resp = await request.send();
//------Read response
String result = await resp.stream.bytesToString();
//-------Your response
print(result);
Hopefully this gives you a good overview how I solved the problem.

Question mark converted to %3F in URI

I'm working on a project and I'm trying to get information from an API. When I write the link it doesn't detect the character "?" and it substitutes this char for "%3F" so I can't access to the API.
final String _charactersUrl = '/api/character/?page=2';
I get status code 500 from the API:
https://rickandmortyapi.com/api/character/%3Fpage=3
The class that gets information from the API
class Api {
final String _baseUrl = 'rickandmortyapi.com';
final String _charactersUrl = '/api/character/?page=2';
final String _charactersJsonKey = 'results';
final HttpClient _httpClient = HttpClient();
Future<List<Character>> getCharacters() async {
final uri = Uri.https(_baseUrl, _charactersUrl);
final response = await _getJson(uri);
if (response == null || response[_charactersJsonKey] == null) {
print('Api.getCharacters(): Error while retrieving characters');
return null;
}
return _convert(response[_charactersJsonKey]);
}
Future<Map<String, dynamic>> _getJson(Uri uri) async {
try {
final request = await _httpClient.getUrl(uri);
final response = await request.close();
if (response.statusCode != HttpStatus.OK) {
print('Api._getJson($uri) status code is ${response.statusCode}');
return null;
}
final responseBody = await response.transform(utf8.decoder).join();
return json.decode(responseBody);
} on Exception catch (e) {
print('Api._getJson($uri) exception thrown: $e');
return null;
}
}
List<Character> _convert(List charactersJson) {
List<Character> characters = <Character>[];
charactersJson.forEach((character) {
characters.add(Character.fromJson(character));
});
return characters;
}
}
I would be very grateful if someone could help me. Thanks!
The Uri class expects you to use the Uri.https constructor differently.
The third positional parameter is queryParameters, which you should use instead of passing your query parameters to the unencodedPath:
final String _baseUrl = 'rickandmortyapi.com';
final String _charactersPath = '/api/character/';
final Map<String, String> _queryParameters = <String, String>{
'page': '2',
};
Future<List<Character>> getCharacters() async {
final uri = Uri.https(_baseUrl, _charactersPath, _queryParameters);
...