How can I specify a GridFS bucket? - mongodb

This is my express.js code to upload and download files to GridFS:
var fs = require("fs");
var gridStream = require("gridfs-stream");
var mongoose = require("mongoose");
exports.init = function(app, db)
{
var grid = gridStream(db, mongoose.mongo);
app.post("/UploadFile", function(request, response)
{
var file = request.files.UploadedFile;
var meta = request.param("Meta");
var name = request.param("Name");
var stream = grid.createWriteStream(
{
filename: name,
metadata: meta
});
fs.createReadStream(file.path)
.on("end", function()
{
response.send({ Success: true });
})
.on("Error", function(error)
{
HandleError(error, response);
})
.pipe(stream);
});
app.get("/DownloadFile", function(request, response)
{
var selector = request.param("Selector");
response.writeHead(200, { "Content-Type" : "image/png"});
grid.createReadStream({ filename: "FileUploadNamed" }).pipe(response);
});
}
It works perfectly, but I'd like to specify a bucket to read and write from, but I'm not sure how to do that. I've seen examples online calling a GridFS constructor, but as you can see I'm not doing that here. The documentation also says that it's possible to supply a different bucket name, but I don't see anything on how.
How can I select which bucket that my files are saved to and read from?

This is not well documented in gridfs-stream or the underlying native mongodb driver it uses, but here is how you do it:
Here is the options object from the gridfs-stream createWriteStream example (note the root option):
{
_id: '50e03d29edfdc00d34000001',
filename: 'my_file.txt',
mode: 'w',
chunkSize: 1024,
content_type: 'plain/text',
root: 'my_collection', // Bucket will be 'my_collection' instead of 'fs'
metadata: {
...
}
}
Why it works:
gridfs-stream passes through the options object you pass a call to createWriteStream or createReadStream to the underlying mongodb driver to create a gridStore object to represent the file. The mongodb driver in turn recognizes root in the options object as an override of the default "fs" grid bucket prefix string.

Related

lokijs storing in filesystem not in in-memeory

I tried the following basic example from the online;
var loki = require('lokijs');
var lokiDatabase = new loki('oops.json');
var collection = lokiDatabase.addCollection('props',{
indices: ['name']
});
function hello() {
var insertjson = {name:'Ram'};
collection.insert(insertjson);
lokiDatabase.saveDatabase();
var select = collection.findOne({name:'Ram'});
console.log('select response:'+ select.name);
}
hello();
I am able to get the output with findOne method;
But here, the question is; as tutorials said, LokiJS is an in-memory database; wheras, I can see all the inserts & updates are presenting in the oops.json file.
Where we are storing/from to in-memory here?
Did I understood the concepts wrong?
enter image description here
//lokirouter.js
const db=require('./lokidb')
const router=require('express').Router
class Erouter{
static get(){
router.get("/",(req,res)=>{
db.loadDatabase({},function(){
try{
const data=db.getCollection('items').find({})
res.send(data).status(200)
}
catch(r){
res.status(500).send(`${r}`)
}
})
})
router.post("/",(req,res)=>{
db.loadDatabase({},()=>{
try{
const data=db.getCollection('items').insert(req.body.criteria)
db.saveDatabase(data)
db.save(data)
res.send(data).status(200)
}
catch(r){
res.status(500).send(`${r}`)
}
})
})
return router
}}
module.exports=Erouter
//lokidb.js
var loki=require('lokijs')
var db = new loki('loki.db');
var items = db.addCollection('items');
module.exports=db
//lokiapp.js
const lokirouter=require('./lokirouter')
const express =require("express")
const bodyParser = require('body-parser')
const app=express()
const db=require('./lokidb')
const port=8000;
app.listen(port)
console.log("Server Started"+ " "+port)
app.use(bodyParser.urlencoded({ extended: true }))
app.use(bodyParser.json())
app.use("/",lokirouter.get())
Lokijs is a document driven database,besides not necessary to store records only a json file,can also store in local database file creating local_database.db for instance.
As previous mentioned answer below you have to run it using postman.
when you insert records in request body in json format
ex:{ "criteria":{"name":"jason"} } it will get inserted into the local_database.db file.Similarly to retrive the records you can call get api. Since to find a particular record you can use findOne({name:"jason"}).

Using Sailsjs Skipper file uploading with Flowjs

I'm trying to use skipper and flowjs with ng-flow together for big file uploading.
Based on sample for Nodejs located in flowjs repository, I've created my sails controller and service to handle file uploads. When I uploading a small file it's works fine, but if I try to upload bigger file (e.g. video of 200 Mb) I'm receiving errors (listed below) and array req.file('file')._files is empty. Intersting fact that it happening only few times during uploading. For example, if flowjs cut the file for 150 chunks, in sails console these errors will appear only 3-5 times. So, almost all chunks will uploaded to the server, but a few are lost and in result file is corrupted.
verbose: Unable to expose body parameter `flowChunkNumber` in streaming upload! Client tried to send a text parameter (flowChunkNumber) after one or more files had already been sent. Make sure you always send text params first, then your files.
These errors appears for all flowjs parameters.
I know about that text parameters must be sent first for correct work with skipper. And in chrome network console I've checked that flowjs sends this data in a correct order.
Any suggestions?
Controller method
upload: function (req, res) {
flow.post(req, function (status, filename, original_filename, identifier) {
sails.log.debug('Flow: POST', status, original_filename, identifier);
res.status(status).send();
});
}
Service post method
$.post = function(req, callback) {
var fields = req.body;
var file = req.file($.fileParameterName);
if (!file || !file._files.length) {
console.log('no file', req);
file.upload(function() {});
}
var stream = file._files[0].stream;
var chunkNumber = fields.flowChunkNumber;
var chunkSize = fields.flowChunkSize;
var totalSize = fields.flowTotalSize;
var identifier = cleanIdentifier(fields.flowIdentifier);
var filename = fields.flowFilename;
if (file._files.length === 0 || !stream.byteCount)
{
callback('invalid_flow_request', null, null, null);
return;
}
var original_filename = stream.filename;
var validation = validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename, stream.byteCount);
if (validation == 'valid')
{
var chunkFilename = getChunkFilename(chunkNumber, identifier);
// Save the chunk by skipper file upload api
file.upload({saveAs:chunkFilename},function(err, uploadedFiles){
// Do we have all the chunks?
var currentTestChunk = 1;
var numberOfChunks = Math.max(Math.floor(totalSize / (chunkSize * 1.0)), 1);
var testChunkExists = function()
{
fs.exists(getChunkFilename(currentTestChunk, identifier), function(exists)
{
if (exists)
{
currentTestChunk++;
if (currentTestChunk > numberOfChunks)
{
callback('done', filename, original_filename, identifier);
} else {
// Recursion
testChunkExists();
}
} else {
callback('partly_done', filename, original_filename, identifier);
}
});
};
testChunkExists();
});
} else {
callback(validation, filename, original_filename, identifier);
}};
Edit
Found solution to set flowjs property maxChunkRetries: 5, because by default it's 0.
On the server side, if req.file('file')._files is empty I'm throwing not permanent(in context of flowjs) error.
So, it's solves my problem, but question why it behave like this is still open. Sample code for flowjs and Nodejs uses connect-multiparty and has no any additional error handling code, so it's most likely skipper bodyparser bug.

Read file from GridFS based on _ID using mongoskin & nodejs

I am using mongoskin in my nodejs based application. I have used GridFS to uplaod the file. I am able to upload and read it back using the "filename" however I want to read it back using _id. How can i do? Following are code details.
Working code to read the file based on filename:
exports.previewFile = function (req, res) {
var contentId = req.params.contentid;
var gs = DBModule.db.gridStore('69316_103528209714703_155822_n.jpg', 'r');
gs.read(function (err, data) {
if (!err) {
res.setHeader('Content-Type', gs.contentType);
res.end(data);
} else {
log.error({err: err}, 'Failed to read the content for id '+contentId);
res.status(constants.HTTP_CODE_INTERNAL_SERVER_ERROR);
res.json({error: err});
}
});
};
How this code can be modified to make it work based on id?
After few hit & trial following code works. This is surprise bcz input parameter seems searching on all the fields.
//view file from database
exports.previewContent = function (req, res) {
var contentId = new DBModule.BSON.ObjectID(req.params.contentid);
console.log('Calling previewFile inside FileUploadService for content id ' + contentId);
var gs = DBModule.db.gridStore(contentId, 'r');
gs.read(function (err, data) {
if (!err) {
//res.setHeader('Content-Type', metadata.contentType);
res.end(data);
} else {
log.error({err: err}, 'Failed to read the content for id ' + contentId);
res.status(constants.HTTP_CODE_INTERNAL_SERVER_ERROR);
res.json({error: err});
}
});
};

Store file in Mongo's GridFS with ExpressJS after upload

I have started building a REST api using expressJS. I am new to node so please bear with me. I want to be able to let users upload a file directly to Mongo's GridFS using a post to the /upload route.
From what I understand in expressJS documentation the req.files.image object is available in the route after uploading, which also includes a path and filename attribute. But how can I exactly read the image data and store it into GridFS?
I have looked into gridfs-stream but I can't tie ends together. Do I first need to read the file and then use that data for the writestream pipe? Or can I just use the file object from express and use those attributes to construct a writestream? Any pointers would be appreciated!
Here's a simple demo:
var express = require('express');
var fs = require('fs');
var mongo = require('mongodb');
var Grid = require('gridfs-stream');
var db = new mongo.Db('test', new mongo.Server("127.0.0.1", 27017), { safe : false });
db.open(function (err) {
if (err) {
throw err;
}
var gfs = Grid(db, mongo);
var app = express();
app.use(express.bodyParser());
app.post('/upload', function(req, res) {
var tempfile = req.files.filename.path;
var origname = req.files.filename.name;
var writestream = gfs.createWriteStream({ filename: origname });
// open a stream to the temporary file created by Express...
fs.createReadStream(tempfile)
.on('end', function() {
res.send('OK');
})
.on('error', function() {
res.send('ERR');
})
// and pipe it to gfs
.pipe(writestream);
});
app.get('/download', function(req, res) {
// TODO: set proper mime type + filename, handle errors, etc...
gfs
// create a read stream from gfs...
.createReadStream({ filename: req.param('filename') })
// and pipe it to Express' response
.pipe(res);
});
app.listen(3012);
});
I use httpie to upload a file:
http --form post localhost:3012/upload filename#~/Desktop/test.png
You can check your database if the file is uploaded:
$ mongofiles list -d test
connected to: 127.0.0.1
test.png 5520
You can also download it again:
http --download get localhost:3012/download?filename=test.png

How to use GridFS to store images using Node.js and Mongoose

I am new to Node.js. Can anyone provide me an example of how to use GridFS for storing and retrieving binary data, such as images, using Node.js and Mongoose? Do I need to directly access GridFS?
I was not satisfied with the highest rated answer here and so I'm providing a new one:
I ended up using the node module 'gridfs-stream' (great documentation there!) which can be installed via npm.
With it, and in combination with mongoose, it could look like this:
var fs = require('fs');
var mongoose = require("mongoose");
var Grid = require('gridfs-stream');
var GridFS = Grid(mongoose.connection.db, mongoose.mongo);
function putFile(path, name, callback) {
var writestream = GridFS.createWriteStream({
filename: name
});
writestream.on('close', function (file) {
callback(null, file);
});
fs.createReadStream(path).pipe(writestream);
}
Note that path is the path of the file on the local system.
As for my read function of the file, for my case I just need to stream the file to the browser (using express):
try {
var readstream = GridFS.createReadStream({_id: id});
readstream.pipe(res);
} catch (err) {
log.error(err);
return next(errors.create(404, "File not found."));
}
Answers so far are good, however, I believe it would be beneficial to document here how to do this using the official mongodb nodejs driver instead of relying on further abstractions such as "gridfs-stream".
One previous answer has indeed utilized the official mongodb driver, however they use the Gridstore API; which has since been deprecated, see here. My example will be using the new GridFSBucket API.
The question is quite broad as such my answer will be an entire nodejs program. This will include setting up the express server, mongodb driver, defining the routes and handling the GET and POST routes.
Npm Packages Used
express (nodejs web application framework to simplify this snippet)
multer (for handling multipart/form-data requests)
mongodb (official mongodb nodejs driver)
The GET photo route takes a Mongo ObjectID as a parameter to retrieve the image.
I configure multer to keep the uploaded file in memory. This means the photo file will not be written to the file system at anytime, and instead be streamed straight from memory into GridFS.
/**
* NPM Module dependencies.
*/
const express = require('express');
const photoRoute = express.Router();
const multer = require('multer');
var storage = multer.memoryStorage()
var upload = multer({ storage: storage, limits: { fields: 1, fileSize: 6000000, files: 1, parts: 2 }});
const mongodb = require('mongodb');
const MongoClient = require('mongodb').MongoClient;
const ObjectID = require('mongodb').ObjectID;
let db;
/**
* NodeJS Module dependencies.
*/
const { Readable } = require('stream');
/**
* Create Express server && Routes configuration.
*/
const app = express();
app.use('/photos', photoRoute);
/**
* Connect Mongo Driver to MongoDB.
*/
MongoClient.connect('mongodb://localhost/photoDB', (err, database) => {
if (err) {
console.log('MongoDB Connection Error. Please make sure that MongoDB is running.');
process.exit(1);
}
db = database;
});
/**
* GET photo by ID Route
*/
photoRoute.get('/:photoID', (req, res) => {
try {
var photoID = new ObjectID(req.params.photoID);
} catch(err) {
return res.status(400).json({ message: "Invalid PhotoID in URL parameter. Must be a single String of 12 bytes or a string of 24 hex characters" });
}
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'photos'
});
let downloadStream = bucket.openDownloadStream(photoID);
downloadStream.on('data', (chunk) => {
res.write(chunk);
});
downloadStream.on('error', () => {
res.sendStatus(404);
});
downloadStream.on('end', () => {
res.end();
});
});
/**
* POST photo Route
*/
photoRoute.post('/', (req, res) => {
upload.single('photo')(req, res, (err) => {
if (err) {
return res.status(400).json({ message: "Upload Request Validation Failed" });
} else if(!req.body.name) {
return res.status(400).json({ message: "No photo name in request body" });
}
let photoName = req.body.name;
// Covert buffer to Readable Stream
const readablePhotoStream = new Readable();
readablePhotoStream.push(req.file.buffer);
readablePhotoStream.push(null);
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'photos'
});
let uploadStream = bucket.openUploadStream(photoName);
let id = uploadStream.id;
readablePhotoStream.pipe(uploadStream);
uploadStream.on('error', () => {
return res.status(500).json({ message: "Error uploading file" });
});
uploadStream.on('finish', () => {
return res.status(201).json({ message: "File uploaded successfully, stored under Mongo ObjectID: " + id });
});
});
});
app.listen(3005, () => {
console.log("App listening on port 3005!");
});
I wrote a blog post on this subject; is is an elaboration of my answer. Available here
Further Reading/Inspiration:
NodeJs Streams: Everything you need to know
Multer NPM docs
Nodejs MongoDB Driver
I suggest taking a look at this question: Problem with MongoDB GridFS Saving Files with Node.JS
Copied example from the answer (credit goes to christkv):
// You can use an object id as well as filename now
var gs = new mongodb.GridStore(this.db, filename, "w", {
"chunk_size": 1024*4,
metadata: {
hashpath:gridfs_name,
hash:hash,
name: name
}
});
gs.open(function(err,store) {
// Write data and automatically close on finished write
gs.writeBuffer(data, true, function(err,chunk) {
// Each file has an md5 in the file structure
cb(err,hash,chunk);
});
});
It looks like the writeBuffer has since been deprecated.
/Users/kmandrup/private/repos/node-mongodb-native/HISTORY:
82 * Fixed dereference method on Db class to correctly dereference Db reference objects.
83 * Moved connect object onto Db class(Db.connect) as well as keeping backward compatibility.
84: * Removed writeBuffer method from gridstore, write handles switching automatically now.
85 * Changed readBuffer to read on Gridstore, Gridstore now only supports Binary Buffers no Strings anymore.
remove the fileupload library
and if it is giving some multi-part header related error than remove the content-type from the headers