Is there a nodejs module or example of how to make GridFS files accessible with WebDAV? - mongodb

I have an existing node.js app where users have a library of files that are stored with GridFS. Each user has their own library. I would like to make the library mountable with WebDAV so that a user could manage their library from their desktop.
I have seen jsDAV used to access the filesystem but it is not clear how to extend it for use with a virtual file system. I found gitDav but it is not clear how to use it.
Is this even possible without starting from scratch?

I was looking to use jsDAV to make some resources available through WebDAV. Failing to find a working example, I studied the comments in the source and wrote one myself. jsDAV is a port from a PHP library. The Sabre manual is useful guide in general. One thing to remember is that since we're in an asynchronous environment, functions that return the results in PHP might have to invoke a callback function instead. This usually happens when the operation in question involves reading from the disk. The first parameter to the callback will always be an error object, which should be null when all goes well.
'use strict';
var crypto = require('crypto');
var jsDAV = require("jsDAV/lib/jsdav");
var jsDAVLocksBackendFS = require("jsDAV/lib/DAV/plugins/locks/fs");
var jsDAVFile = require("jsDAV/lib/DAV/file");
var jsDAVCollection = require("jsDAV/lib/DAV/collection");
var jsExceptions = require("jsDAV/lib/shared/exceptions");
var VirtualFile = jsDAVFile.extend(
{
initialize: function(name, buffer) {
this.name = name;
this.buffer = buffer;
},
getName: function() {
return this.name;
},
get: function(callback) {
callback(null, this.buffer);
},
put: function(data, type, callback) {
callback(new jsExceptions.Forbidden("Permission denied to change data"));
},
getSize: function(callback) {
callback(null, this.buffer.length);
},
getETag: function(callback) {
var shasum = crypto.createHash('sha1');
shasum.update(this.buffer);
var etag = '"' + shasum.digest('hex') + '"';
callback(null, etag);
},
getContentType: function(callback) {
callback(null, 'text/plain');
}
});
var VirtualDirectory = jsDAVCollection.extend(
{
initialize: function(name, children) {
this.name = name;
this.children = children;
},
getChildren: function(callback) {
var list = [];
for (var name in this.children) {
list.push(this.children[name]);
}
callback(null, list);
},
getChild: function(name, callback) {
var child = this.children[name];
if (child) {
callback(null, child);
} else {
callback(new jsExceptions.NotFound("File not found"));
}
},
childExists: function(name, callback) {
var exists = (this.children[name] !== undefined);
callback(null, exists);
},
getName: function() {
return this.name;
}
});
var children = {};
for (var i = 1; i <= 10; i++) {
var name = 'file' + i + '.txt';
var text = 'Hello world, #' + i;
children[name] = VirtualFile.new(name, new Buffer(text, 'utf8'));
}
var grandchildren = {};
for (var i = 66; i <= 99; i++) {
var name = 'beer' + i + '.txt';
var text = i + ' bottles of beer';
grandchildren[name] = VirtualFile.new(name, new Buffer(text, 'utf8'));
}
children['folder'] = VirtualDirectory.new('folder', grandchildren);
var root = VirtualDirectory.new(null, children);
var options = {
node: root,
locksBackend: jsDAVLocksBackendFS.new(__dirname + "/data")
};
var port = 8000;
jsDAV.createServer(options, port);

It looks like jsDAV is the only option. It is a port of a PHP library and it is not setup in such a way that you can use it like a normal node.js module. I found a few examples of server types that others have created to connect it with dropbox and couchdb.
I am now working on a server type that will work more like you would expect a node.js module to work. The next step will be making it play nice with npm. You can see my fork here.

Related

Protractor POM method is not recognizing

spec.js
describe('Testing an animal adoption flow using page object', function() {
beforeEach(function() {
browser.get('http://www.thetestroom.com/jswebapp/index.html');
});
var home_page = require('./pages/home_page.js');
it ('Should be able to adopt an animal by page object', function() {
home_page.enterName('Blabla');
expect(home_page.getDynamicText()).toBe('Blabla');
var animal_page = home_page.clickContinue();
animal_page.selectAnimal(1);
var confirm_page = animal_page.clickContinue();
expect(confirm_page.getTitle()).toContain('Thank');
});
});
home_page.js
require('./animal_page.js');
var home_page = function() {
this.nameTextBox = element(by.model('person.name'));
this.dynamicText = element(by.binding('person.name'));
this.continueButton = element(by.buttonText('CONTINUE'));
this.enterName = function(name) {
this.nameTextBox.sendKeys(name);
};
this.getDynamicText = function() {
return this.dynamicText.getText();
};
this.clickContinue = function() {
this.continueButton.click();
return require('./animal_page.js');
};
};
Failures:
Testing an animal adoption flow using page object Should be able to adopt an animal by page object
Message:
[31m Failed: home_page.enterName is not a function[0m
Stack:
TypeError: home_page.enterName is not a function
You don't create an instance of your constructor function with new keyword. It should have been
var home_page = new (require('./pages/home_page.js'));
and you need to instruct js what you are exporting, so your home page should be
require('./animal_page.js');
var home_page = function() {
this.nameTextBox = element(by.model('person.name'));
this.dynamicText = element(by.binding('person.name'));
this.continueButton = element(by.buttonText('CONTINUE'));
this.enterName = function(name) {
this.nameTextBox.sendKeys(name);
};
this.getDynamicText = function() {
return this.dynamicText.getText();
};
this.clickContinue = function() {
this.continueButton.click();
return require('./animal_page.js');
};
}
module.exports = home_page; // <------ this line
but make sure you do the same with animal_page
I got the answer, we need to include
spec.js
const { browser } = require('protractor');
home_page.js
module.exports = new home_page();

ProxyConnectionError when connecting as Anonymous

I have developed an operator to retrieve information from Orion Context Broker.
It works perfectly when I'm loggin but if I try to enter as anonymous (with the embedded URL) in a incognito window, the operator raises the next error:
(link to the image): http://i.stack.imgur.com/jxMkr.png
This is the code:
var doInitialSubscription = function doInitialSubscription() {
this.subscriptionId = null;
this.ngsi_server = MashupPlatform.prefs.get('ngsi_server');
this.ngsi_proxy = MashupPlatform.prefs.get('ngsi_proxy');
this.connection = new NGSI.Connection(this.ngsi_server, {
ngsi_proxy_url: this.ngsi_proxy
});
console.log("Send initial subscription");
var types = ['SMARTMETER'];
var entityIdList = [];
var entityId;
entityId = {
id: '.*',
type: 'SMARTMETER',
isPattern: true
};
entityIdList.push(entityId);
var attributeList = null;
var duration = 'PT3H';
var throttling = null;
var notifyConditions = [{
'type': 'ONCHANGE',
'condValues': condValues
}];
var options = {
flat: true,
onNotify: handlerReceiveEntity.bind(this),
onSuccess: function (data) {
console.log("Subscription success ID: "+data.subscriptionId);
this.subscriptionId = data.subscriptionId;
this.refresh_interval = setInterval(refreshNGSISubscription.bind(this), 1000 * 60 * 60 * 2); // each 2 hours
window.addEventListener("beforeunload", function () {
this.connection.cancelSubscription(this.subscriptionId);
}.bind(this));
}.bind(this),
onFailure: function(data) {
console.log(data);
}
};
console.log("Now creating subscription...");
this.connection.createSubscription(entityIdList, attributeList, duration, throttling, notifyConditions, options);
};
Any idea of what is wrong?
According to user comments on the question, updating to Orion 0.19.0 (following the DB upgrade procedure detailed here) solves the problem.

With Chrome FileSystem, let the user choose a Directory, load files inside. And save files in that directory without prompting again

I could not found any examples with this scenario so here we go:
I want the user choose a directory, load all files inside it, change them, and save this file overriding it or saving a new file in that same directory without asking where he want to save.
I don't know how to list the files of the directory
I don't know how to save a file in a directory without prompting the filechooser window
I believe it is possible because I see something similar here (last paragraph):
http://www.developer.com/lang/using-the-file-api-outside-the-sandbox-in-chrome-packaged-apps.html
Any answer will be appreciated, Thank you
EDIT: Thanks to Chris Johnsen for giving me this great answer:
var fileHandler = function() {
var _entry = null;
this.open = function(cb) {
chrome.fileSystem.chooseEntry({
type: 'openDirectory'
}, function(dirEntry) {
if (!dirEntry || !dirEntry.isDirectory) {
cb && cb(null);
return;
}
_entry = dirEntry;
listDir(_entry, cb);
});
};
this.save = function(filename, source) {
chrome.fileSystem.getWritableEntry(_entry, function(entry) {
entry.getFile(filename, {
create: true
}, function(entry) {
entry.createWriter(function(writer) {
writer.onwrite = function() {
writer.onwrite = null;
writer.truncate(writer.position);
};
writer.write(new Blob([source], {
type: 'text/javascript'
}));
});
});
});
};
this.saveAs = function(filename, source) {
chrome.fileSystem.chooseEntry({
type: 'openDirectory'
}, function(entry) {
chrome.fileSystem.getWritableEntry(entry, function(entry) {
entry.getFile(filename, {
create: true
}, function(entry) {
entry.createWriter(function(writer) {
writer.onwrite = function() {
writer.onwrite = null;
writer.truncate(writer.position);
};
writer.write(new Blob([source], {
type: 'text/javascript'
}));
});
});
});
});
};
var listDir = function(dirent, cb, listing) {
if (listing === undefined) {
listing = [];
}
var reader = dirent.createReader();
var read_some = reader.readEntries.bind(reader, function(ents) {
if (ents.length === 0) {
return cb && cb(listing);
}
var process_some = function(ents, i) {
for (; i < ents.length; i++) {
listing.push(ents[i]);
if (ents[i].isDirectory) {
return listDir(ents[i], process_some.bind(null, ents, i + 1), listing);
}
}
read_some();
};
process_some(ents, 0);
}, function() {
console.error('error reading directory');
});
read_some();
};
};
Your save method should work fine (mostly, see below) for your second requirement (write to a code-chosen filename without another user prompt), but there are a couple of bugs in open (at least as presented in the question):
Inside the chooseEntry callback, this !== fileHandler because the callback is invoked with a different this (probably the background page’s window object).
You can work around this in several ways:
Use fileHandler instead of this (if you are not using it as any kind of prototype).
Use .bind(this) to bind each of your callback functions to the same context.
Use var self = this; at the top of open and use self.entry (et cetera) in the callbacks.
You may want to call cb for the success case. Maybe you have another way of postponing calls to (e.g.) fileHandler.save (clicking on some element to trigger the save?), but adding something like
⋮
cb && cb(self.entry);
⋮
after self.entry = dirEntry makes it easy to (e.g.) chain open and save:
fileHandler.open(function(ent) {
fileHandler.save('newfile','This is the text\nto save in the (possibly) new file.');
});
There is a latent bug in save: if you ever overwrite an existing file, then you will want to call writer.truncate() (unless you always write more bytes than the file originally held).
⋮
writer.onwrite = function() {
writer.onwrite = null;
writer.truncate(writer.position);
};
writer.write(…);
⋮
It looks like you have a good start on the file listing part. If you want to reference the list of files later, then you might want to save them in your object instead of just logging them; this can get a bit hairy if you want to recurse into subdirectories (and also not assume that readEntries returns everything for its first call).
function list_dir(dirent, cb, listing) {
if (listing === undefined) listing = [];
var reader = dirent.createReader();
var read_some = reader.readEntries.bind(reader, function(ents) {
if (ents.length === 0)
return cb && cb(listing);
process_some(ents, 0);
function process_some(ents, i) {
for(; i < ents.length; i++) {
listing.push(ents[i]);
if (ents[i].isDirectory)
return list_dir(ents[i], process_some.bind(null, ents, i + 1), listing);
}
read_some();
}
}, function() {
console.error('error reading directory');
});
read_some();
}
You could use it in the open callback (assuming you add its success callback) like this:
fileHandler.open(function(ent) {
ent && list_dir(ent, function(listing) {
fileHandler.listing = listing;
console.log('listing', fileHandler.listing.map(function(ent){return ent.fullPath}).join('\n'));
fileHandler.save('a_dir/somefile','This is some data.');
});
});

Creating new Meteor collections on the fly

Is it possible to create new Meteor collections on-the-fly? I'd like to create foo_bar or bar_bar depending on some pathname which should be a global variable I suppose (so I can access it throughout my whole application).
Something like:
var prefix = window.location.pathname.replace(/^\/([^\/]*).*$/, '$1');
var Bar = new Meteor.Collection(prefix+'_bar');
The thing here is that I should get my prefix variable from URL, so if i declare it outside of if (Meteor.isClient) I get an error: ReferenceError: window is not defined. Is it possible to do something like that at all?
Edit : Using the first iteration of Akshats answer my project js : http://pastie.org/6411287
I'm not entirely certain this will work:
You need it in two pieces, the first to load collections you've set up before (on both the client and server)
var collections = {};
var mysettings = new Meteor.Collection('settings') //use your settings
//Startup
Collectionlist = mysettings.find({type:'collection'});
Collectionlist.forEach(function(doc) {
collections[doc.name] = new Meteor.Collection(doc.name);
})'
And you need a bit to add the collections on the server:
Meteor.methods({
'create_server_col' : function(collectionname) {
mysettings.insert({type:'collection', name: collectionname});
newcollections[collectionname] = new Collection(collectionname);
return true;
}
});
And you need to create them on the client:
//Create the collection:
Meteor.call('create_server_col', 'My New Collection Name', function(err,result) {
if(result) {
alert("Collection made");
}
else
{
console.log(err);
}
}
Again, this is all untested so I'm just giving it a shot hopefully it works.
EDIT
Perhaps the below should work, I've added a couple of checks to see if the collection exists first. Please could you run meteor reset before you use it to sort bugs from the code above:
var collections = {};
var mysettings = new Meteor.Collection('settings')
if (Meteor.isClient) {
Meteor.startup(function() {
Collectionlist = mysettings.find({type:'collection'});
Collectionlist.forEach(function(doc) {
eval("var "+doc.name+" = new Meteor.Collection("+doc.name+"));
});
});
Template.hello.greeting = function () {
return "Welcome to testColl.";
};
var collectionname=prompt("Enter a collection name to create:","collection name")
create_collection(collectionname);
function create_collection(name) {
Meteor.call('create_server_col', 'tempcoll', function(err,result) {
if(!err) {
if(result) {
//make sure name is safe
eval("var "+name+" = new Meteor.Collection('"+name+"'));
alert("Collection made");
console.log(result);
console.log(collections);
} else {
alert("This collection already exists");
}
}
else
{
alert("Error see console");
console.log(err);
}
});
}
}
if (Meteor.isServer) {
Meteor.startup(function () {
// code to run on server at startup
Collectionlist = mysettings.find({type:'collection'});
Collectionlist.forEach(function(doc) {
collections[doc.name] = new Meteor.Collection(doc.name);
});
});
Meteor.methods({
'create_server_col' : function(collectionname) {
if(!mysettings.findOne({type:'collection', name: collectionname})) {
mysettings.insert({type:'collection', name: collectionname});
collections[collectionname] = new Meteor.Collection(collectionname);
return true;
}
else
{
return false; //Collection already exists
}
}
});
}
Also make sure your names are javascript escaped.
Things got much easier:
var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
db.createCollection("COLLECTION_NAME", (err, res) => {
console.log(res);
});
Run this in your server method.

Mongodb inserts not completing successfully (using node.js)

I've got a node.js script that loads an XML file. It loops through each element in the Mongo array and says that they're all getting inserted correctly, but when the script has completed a check of db.collection.count(); tells me that far fewer records have been inserted into the database than the number expected.
How can I make mongo and node.js play nicely with inserts?
GrabRss = function() {
var http = require('http');
var sys = require('sys');
var xml2js = require('xml2js');
var fs = require('fs');
var Db = require('../lib/mongodb').Db,
Conn = require('../lib/mongodb').Connection,
Server = require('../lib/mongodb').Server,
// BSON = require('../lib/mongodb').BSONPure;
BSON = require('../lib/mongodb').BSONNative;
var data;
var checked = 0;
var len = 0;
GotResponse = function(res) {
var ResponseBody = "";
res.on('data', DoChunk);
res.on('end', EndResponse);
function DoChunk(chunk){
ResponseBody += chunk;
}
function EndResponse() {
//console.log(ResponseBody);
var parser = new xml2js.Parser();
parser.addListener('end', GotRSSObject);
parser.parseString(ResponseBody);
}
}
GotError = function(e) {
console.log("Got error: " + e.message);
}
GotRSSObject = function(r){
items = r.item;
//console.log(sys.inspect(r));
var db = new Db('myrssdb', new Server('localhost', 27017, {}), {native_parser:false});
db.open(function(err, db){
db.collection('items', function(err, col) {
len = movies.length;
for (i in items) {
SaveItem(items[i], col);
}
});
});
}
SaveItem = function(m, c) {
/* REPLACE FROM HERE IN ANSWER */
c.find({'id': m.id}, function(err, cursor){
cursor.nextObject(function(err, doc) {
if (doc == null) {
c.insert(m, function(err, docs) {
docs.forEach(function(doc) {
console.log('Saved: '+doc.id+' '+doc.keywords);
});
});
} else {
console.log('Skipped: '+m.id);
}
if (++checked >= len) {
process.exit(0);
}
});
});
/* REPLACE TO HERE IN ANSWER */
}
//http.get(options, GotResponse).on('error', GotError);
var x2js = new xml2js.Parser();
fs.readFile('/home/ubuntu/myrss.rss', function(err, data) {
x2js.parseString(data);
});
x2js.addListener('end', GotRSSObject);
}
GrabRss();
As requested, the code is above. The file is read locally (though is used to be an HTTP request, but it's a 25 meg file now, lots of RSS records)
I just ran the file with some ~10k records in it and a count of the items in the mongoDB after the script has run is about 800 items.
As per the answer I replaced the insert code:
with:
c.update({'id': m.id}, {$set: m}, {upsert: true, safe: true}, function(err){
if (err) console.warn(err.message);
else console.log(m.keywords);
if (++checked >= len) {
console.log(len);
//process.exit(0);
process.exit(0);
}
});
By default, MongoDB writes do not check for an error.
You need to set safe:true in the options to your insert, as explained in the documentation for node-mongodb-native:
var collection = new mongodb.Collection(client, 'test_collection');
collection.insert({hello: 'world'}, {safe:true},
function(err, objects) {
if (err) console.warn(err.message);
if (err && err.message.indexOf('E11000 ') !== -1) {
// this _id was already inserted in the database
}
});
Otherwise your callback will not be invoked for errors and your client won't know about them.
You probably also want to look at upserts and updates, as it is incredibly inefficient to do find & insert if null in a loop.
Instead, upsert will update if the matching document exists, otherwise it will insert a new one. An explanation on how to do this in Node is in the documentaiton for the driver.