Store file in Mongo's GridFS with ExpressJS after upload - mongodb

I have started building a REST api using expressJS. I am new to node so please bear with me. I want to be able to let users upload a file directly to Mongo's GridFS using a post to the /upload route.
From what I understand in expressJS documentation the req.files.image object is available in the route after uploading, which also includes a path and filename attribute. But how can I exactly read the image data and store it into GridFS?
I have looked into gridfs-stream but I can't tie ends together. Do I first need to read the file and then use that data for the writestream pipe? Or can I just use the file object from express and use those attributes to construct a writestream? Any pointers would be appreciated!

Here's a simple demo:
var express = require('express');
var fs = require('fs');
var mongo = require('mongodb');
var Grid = require('gridfs-stream');
var db = new mongo.Db('test', new mongo.Server("127.0.0.1", 27017), { safe : false });
db.open(function (err) {
if (err) {
throw err;
}
var gfs = Grid(db, mongo);
var app = express();
app.use(express.bodyParser());
app.post('/upload', function(req, res) {
var tempfile = req.files.filename.path;
var origname = req.files.filename.name;
var writestream = gfs.createWriteStream({ filename: origname });
// open a stream to the temporary file created by Express...
fs.createReadStream(tempfile)
.on('end', function() {
res.send('OK');
})
.on('error', function() {
res.send('ERR');
})
// and pipe it to gfs
.pipe(writestream);
});
app.get('/download', function(req, res) {
// TODO: set proper mime type + filename, handle errors, etc...
gfs
// create a read stream from gfs...
.createReadStream({ filename: req.param('filename') })
// and pipe it to Express' response
.pipe(res);
});
app.listen(3012);
});
I use httpie to upload a file:
http --form post localhost:3012/upload filename#~/Desktop/test.png
You can check your database if the file is uploaded:
$ mongofiles list -d test
connected to: 127.0.0.1
test.png 5520
You can also download it again:
http --download get localhost:3012/download?filename=test.png

Related

lokijs storing in filesystem not in in-memeory

I tried the following basic example from the online;
var loki = require('lokijs');
var lokiDatabase = new loki('oops.json');
var collection = lokiDatabase.addCollection('props',{
indices: ['name']
});
function hello() {
var insertjson = {name:'Ram'};
collection.insert(insertjson);
lokiDatabase.saveDatabase();
var select = collection.findOne({name:'Ram'});
console.log('select response:'+ select.name);
}
hello();
I am able to get the output with findOne method;
But here, the question is; as tutorials said, LokiJS is an in-memory database; wheras, I can see all the inserts & updates are presenting in the oops.json file.
Where we are storing/from to in-memory here?
Did I understood the concepts wrong?
enter image description here
//lokirouter.js
const db=require('./lokidb')
const router=require('express').Router
class Erouter{
static get(){
router.get("/",(req,res)=>{
db.loadDatabase({},function(){
try{
const data=db.getCollection('items').find({})
res.send(data).status(200)
}
catch(r){
res.status(500).send(`${r}`)
}
})
})
router.post("/",(req,res)=>{
db.loadDatabase({},()=>{
try{
const data=db.getCollection('items').insert(req.body.criteria)
db.saveDatabase(data)
db.save(data)
res.send(data).status(200)
}
catch(r){
res.status(500).send(`${r}`)
}
})
})
return router
}}
module.exports=Erouter
//lokidb.js
var loki=require('lokijs')
var db = new loki('loki.db');
var items = db.addCollection('items');
module.exports=db
//lokiapp.js
const lokirouter=require('./lokirouter')
const express =require("express")
const bodyParser = require('body-parser')
const app=express()
const db=require('./lokidb')
const port=8000;
app.listen(port)
console.log("Server Started"+ " "+port)
app.use(bodyParser.urlencoded({ extended: true }))
app.use(bodyParser.json())
app.use("/",lokirouter.get())
Lokijs is a document driven database,besides not necessary to store records only a json file,can also store in local database file creating local_database.db for instance.
As previous mentioned answer below you have to run it using postman.
when you insert records in request body in json format
ex:{ "criteria":{"name":"jason"} } it will get inserted into the local_database.db file.Similarly to retrive the records you can call get api. Since to find a particular record you can use findOne({name:"jason"}).

URL works until I set it to process.env.MONGOLAB_URI via command line

I've been experimenting with the API of flickr and am looking to deploy the application to Heroku now that I am finished. In order to do this, since I'm using MongoDB, I'm trying to get it to run with mLab. I can log into the shell just fine, I can get the entire thing to run when I use the URL to the database as a string, but suddenly when I assign that URL to process.env.MONGOLAB_URI with SET MONGOLAB_URI="mongodb://dbuser:dbpassword#ds041506.mlab.com:41506/flick
r_image_search" (I'm using Windows 10) in the local command line and try to use it, it stops working. The error message I'm getting is "Error: invalid schema, expected mongodb". Even when I used console.log(url) it returned "mongodb://dbuser:dbpassword#ds041506.mlab.com:41506/flickr_image_search" (I guarantee the username and password I used in the command line are both absolutely correct considering I literally copy+pasted the URL I used that worked into the command line) as I would expect. The IDE I'm using is Visual Studio Code and I haven't been using the heroku mLab add-on to run this but my database on the website itself. I'm out of ideas and can use some help. Here's each relevant file of code I'm using for the application:
app.js
var express = require('express');
var app = express();
var flickr = require('./flickr.js');
var mongo = require('mongodb').MongoClient;
var path = require('path');
var url = process.env.MONGOLAB_URI;
var today = new Date();
var day = today.getDate();
var month = today.getMonth()+1;
var year = today.getFullYear();
var mins = today.getMinutes();
var hr = today.getHours();
today = month + '/' + day + '/' + year + ' ' + hr + ':' + mins;
var obj;
var search;
app.get('/', function(req, res){
res.sendFile(path.join(__dirname + '/index.html'));
});
app.get('/latest', function(req, res){
mongo.connect(url, function(err, db){
if(err) throw err;
var theCollection = db.collection('flickr_image_search');
theCollection.find({}).toArray(function(err, docs){
if(err) throw err;
search = docs;
return docs;
});
db.close();
});
res.send(search);
});
app.get(['/search/:text/:offSet2', '/search/:text'], function(req, res){
var lookInIt = req.params.text;
var listLength = req.params.offSet2;
var searched = flickr.search(lookInIt, listLength || 10);
obj = {
query: lookInIt,
offSet: listLength,
date: today
};
mongo.connect(url, function(err, db){
if(err) throw err;
var imageSearch = db.collection('flickr_image_search');
imageSearch.insert(obj);
db.close();
});
res.send(searched);
});
var port = 3000 || process.env.PORT;
app.listen(port, process.env.IP, function(){
console.log('LISTENING');
});
flickr.js
var private = require('./private');
var Flickr = require("flickrapi"),
flickrOptions = {
api_key: private.key,
secret: private.secret
};
var obj;
function search (query, offSet) {
Flickr.tokenOnly(flickrOptions, function (err, flickr) {
if (err) throw err;
flickr.photos.search({
text: query,
page: 1,
per_page: offSet,
sort: "relevance"
}, function (err, result){
if(err) throw err;
result = result.photos.photo;
for(var key in result){
var picUrl = 'https://farm' + result[key]["farm"] + '.staticflickr.com/' + result[key]["server"] + '/' + result[key].id + '_' + result[key].secret + '.jpg'
result[key].picUrl = picUrl;
var userUrl = 'https://www.flickr.com/photos/' + result[key].owner + '/' + result[key].id;
result[key].userUrl = userUrl;
delete result[key]["ispublic"];
delete result[key]["isfriend"];
delete result[key]['isfamily'];
delete result[key]['id'];
delete result[key]['owner'];
delete result[key]['server'];
delete result[key]['farm'];
delete result[key]['secret'];
}
obj = result;
});
});
return obj;
}
module.exports.search = search;
index.html
<link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet" />
<title>flickr Image Search</title>
<div class='container'>
<h1 class='text-center'>Flickr Image Search</h1>
<h3 class='well'>To get a series of URLs for images, type 'search/whatever-you-want-to-search/the-amount-of-photos-you-want-returned' in the address bar after 'someWebsite.com/'.
If you want to see the most latest searches that were searched, instead of 'search/whatever-you-want-to-search/the-amount-of-photos-you-want-returned' type 'latest'.
When you find the URL that you want, just copy+paste!</h3>
<p class=lead>This application is purely functional, so chances are the queries won't return clean.</p>
<p>This is an application that searches through the photos of Flickr, using its API.</p>
</div>
Heroku uses MONGODB_URI, but you have MONGOLAB_URI.
This could explain why the string works but the environment variable doesn't.
Try typing this in your shell: heroku config
Find the heroku config variable. I would venture to guess it is MONGODB_URI and not MONGOLAB_URI.
If this is the case, reset your environment and heroku config variable to use MONGODB_URI.
You said in a comment: "I'm not sure how to check it on Heroku, especially since the app doesn't want to run on there. Heroku has been giving me troubles when deploying applications in general."
To check your config variables, you can type heroku config in your shell, or you can navigate to your apps dashboard. Click on Settings. Click on Reveal Config Vars.
While you are in the dashboard, you can provision the mLab Add-on that Heroku provides by clicking on the Resources tab. Then in the Add-on's search box, type in mLab MongoDB.
Click on it. This will take you to mLab's GUI where you can view and manage your collections.
If you are wondering what the server setup looks like, see the code below. This will connect to both your local and mLab db where appropriate.
if(process.env.MONGODB_URI) {
mongoose.connect(process.env.MONGODB_URI);
}else {
mongoose.connect(db, function(err){ //db = 'mongodb://localhost/yourdb'
if(err){
console.log(err);
}else {
console.log('mongoose connection is successful on: ' + db);
}
});
}
It's been helpful to me to install dotenv (https://www.npmjs.com/package/dotenv), require('dotenv'), then dotenv.load() before declaring process.env variables. Now stuff in my .env file is read in where it wasn't before. thx.

Read file from GridFS based on _ID using mongoskin & nodejs

I am using mongoskin in my nodejs based application. I have used GridFS to uplaod the file. I am able to upload and read it back using the "filename" however I want to read it back using _id. How can i do? Following are code details.
Working code to read the file based on filename:
exports.previewFile = function (req, res) {
var contentId = req.params.contentid;
var gs = DBModule.db.gridStore('69316_103528209714703_155822_n.jpg', 'r');
gs.read(function (err, data) {
if (!err) {
res.setHeader('Content-Type', gs.contentType);
res.end(data);
} else {
log.error({err: err}, 'Failed to read the content for id '+contentId);
res.status(constants.HTTP_CODE_INTERNAL_SERVER_ERROR);
res.json({error: err});
}
});
};
How this code can be modified to make it work based on id?
After few hit & trial following code works. This is surprise bcz input parameter seems searching on all the fields.
//view file from database
exports.previewContent = function (req, res) {
var contentId = new DBModule.BSON.ObjectID(req.params.contentid);
console.log('Calling previewFile inside FileUploadService for content id ' + contentId);
var gs = DBModule.db.gridStore(contentId, 'r');
gs.read(function (err, data) {
if (!err) {
//res.setHeader('Content-Type', metadata.contentType);
res.end(data);
} else {
log.error({err: err}, 'Failed to read the content for id ' + contentId);
res.status(constants.HTTP_CODE_INTERNAL_SERVER_ERROR);
res.json({error: err});
}
});
};

How can I specify a GridFS bucket?

This is my express.js code to upload and download files to GridFS:
var fs = require("fs");
var gridStream = require("gridfs-stream");
var mongoose = require("mongoose");
exports.init = function(app, db)
{
var grid = gridStream(db, mongoose.mongo);
app.post("/UploadFile", function(request, response)
{
var file = request.files.UploadedFile;
var meta = request.param("Meta");
var name = request.param("Name");
var stream = grid.createWriteStream(
{
filename: name,
metadata: meta
});
fs.createReadStream(file.path)
.on("end", function()
{
response.send({ Success: true });
})
.on("Error", function(error)
{
HandleError(error, response);
})
.pipe(stream);
});
app.get("/DownloadFile", function(request, response)
{
var selector = request.param("Selector");
response.writeHead(200, { "Content-Type" : "image/png"});
grid.createReadStream({ filename: "FileUploadNamed" }).pipe(response);
});
}
It works perfectly, but I'd like to specify a bucket to read and write from, but I'm not sure how to do that. I've seen examples online calling a GridFS constructor, but as you can see I'm not doing that here. The documentation also says that it's possible to supply a different bucket name, but I don't see anything on how.
How can I select which bucket that my files are saved to and read from?
This is not well documented in gridfs-stream or the underlying native mongodb driver it uses, but here is how you do it:
Here is the options object from the gridfs-stream createWriteStream example (note the root option):
{
_id: '50e03d29edfdc00d34000001',
filename: 'my_file.txt',
mode: 'w',
chunkSize: 1024,
content_type: 'plain/text',
root: 'my_collection', // Bucket will be 'my_collection' instead of 'fs'
metadata: {
...
}
}
Why it works:
gridfs-stream passes through the options object you pass a call to createWriteStream or createReadStream to the underlying mongodb driver to create a gridStore object to represent the file. The mongodb driver in turn recognizes root in the options object as an override of the default "fs" grid bucket prefix string.

How to use GridFS to store images using Node.js and Mongoose

I am new to Node.js. Can anyone provide me an example of how to use GridFS for storing and retrieving binary data, such as images, using Node.js and Mongoose? Do I need to directly access GridFS?
I was not satisfied with the highest rated answer here and so I'm providing a new one:
I ended up using the node module 'gridfs-stream' (great documentation there!) which can be installed via npm.
With it, and in combination with mongoose, it could look like this:
var fs = require('fs');
var mongoose = require("mongoose");
var Grid = require('gridfs-stream');
var GridFS = Grid(mongoose.connection.db, mongoose.mongo);
function putFile(path, name, callback) {
var writestream = GridFS.createWriteStream({
filename: name
});
writestream.on('close', function (file) {
callback(null, file);
});
fs.createReadStream(path).pipe(writestream);
}
Note that path is the path of the file on the local system.
As for my read function of the file, for my case I just need to stream the file to the browser (using express):
try {
var readstream = GridFS.createReadStream({_id: id});
readstream.pipe(res);
} catch (err) {
log.error(err);
return next(errors.create(404, "File not found."));
}
Answers so far are good, however, I believe it would be beneficial to document here how to do this using the official mongodb nodejs driver instead of relying on further abstractions such as "gridfs-stream".
One previous answer has indeed utilized the official mongodb driver, however they use the Gridstore API; which has since been deprecated, see here. My example will be using the new GridFSBucket API.
The question is quite broad as such my answer will be an entire nodejs program. This will include setting up the express server, mongodb driver, defining the routes and handling the GET and POST routes.
Npm Packages Used
express (nodejs web application framework to simplify this snippet)
multer (for handling multipart/form-data requests)
mongodb (official mongodb nodejs driver)
The GET photo route takes a Mongo ObjectID as a parameter to retrieve the image.
I configure multer to keep the uploaded file in memory. This means the photo file will not be written to the file system at anytime, and instead be streamed straight from memory into GridFS.
/**
* NPM Module dependencies.
*/
const express = require('express');
const photoRoute = express.Router();
const multer = require('multer');
var storage = multer.memoryStorage()
var upload = multer({ storage: storage, limits: { fields: 1, fileSize: 6000000, files: 1, parts: 2 }});
const mongodb = require('mongodb');
const MongoClient = require('mongodb').MongoClient;
const ObjectID = require('mongodb').ObjectID;
let db;
/**
* NodeJS Module dependencies.
*/
const { Readable } = require('stream');
/**
* Create Express server && Routes configuration.
*/
const app = express();
app.use('/photos', photoRoute);
/**
* Connect Mongo Driver to MongoDB.
*/
MongoClient.connect('mongodb://localhost/photoDB', (err, database) => {
if (err) {
console.log('MongoDB Connection Error. Please make sure that MongoDB is running.');
process.exit(1);
}
db = database;
});
/**
* GET photo by ID Route
*/
photoRoute.get('/:photoID', (req, res) => {
try {
var photoID = new ObjectID(req.params.photoID);
} catch(err) {
return res.status(400).json({ message: "Invalid PhotoID in URL parameter. Must be a single String of 12 bytes or a string of 24 hex characters" });
}
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'photos'
});
let downloadStream = bucket.openDownloadStream(photoID);
downloadStream.on('data', (chunk) => {
res.write(chunk);
});
downloadStream.on('error', () => {
res.sendStatus(404);
});
downloadStream.on('end', () => {
res.end();
});
});
/**
* POST photo Route
*/
photoRoute.post('/', (req, res) => {
upload.single('photo')(req, res, (err) => {
if (err) {
return res.status(400).json({ message: "Upload Request Validation Failed" });
} else if(!req.body.name) {
return res.status(400).json({ message: "No photo name in request body" });
}
let photoName = req.body.name;
// Covert buffer to Readable Stream
const readablePhotoStream = new Readable();
readablePhotoStream.push(req.file.buffer);
readablePhotoStream.push(null);
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'photos'
});
let uploadStream = bucket.openUploadStream(photoName);
let id = uploadStream.id;
readablePhotoStream.pipe(uploadStream);
uploadStream.on('error', () => {
return res.status(500).json({ message: "Error uploading file" });
});
uploadStream.on('finish', () => {
return res.status(201).json({ message: "File uploaded successfully, stored under Mongo ObjectID: " + id });
});
});
});
app.listen(3005, () => {
console.log("App listening on port 3005!");
});
I wrote a blog post on this subject; is is an elaboration of my answer. Available here
Further Reading/Inspiration:
NodeJs Streams: Everything you need to know
Multer NPM docs
Nodejs MongoDB Driver
I suggest taking a look at this question: Problem with MongoDB GridFS Saving Files with Node.JS
Copied example from the answer (credit goes to christkv):
// You can use an object id as well as filename now
var gs = new mongodb.GridStore(this.db, filename, "w", {
"chunk_size": 1024*4,
metadata: {
hashpath:gridfs_name,
hash:hash,
name: name
}
});
gs.open(function(err,store) {
// Write data and automatically close on finished write
gs.writeBuffer(data, true, function(err,chunk) {
// Each file has an md5 in the file structure
cb(err,hash,chunk);
});
});
It looks like the writeBuffer has since been deprecated.
/Users/kmandrup/private/repos/node-mongodb-native/HISTORY:
82 * Fixed dereference method on Db class to correctly dereference Db reference objects.
83 * Moved connect object onto Db class(Db.connect) as well as keeping backward compatibility.
84: * Removed writeBuffer method from gridstore, write handles switching automatically now.
85 * Changed readBuffer to read on Gridstore, Gridstore now only supports Binary Buffers no Strings anymore.
remove the fileupload library
and if it is giving some multi-part header related error than remove the content-type from the headers