lokijs storing in filesystem not in in-memeory - lokijs

I tried the following basic example from the online;
var loki = require('lokijs');
var lokiDatabase = new loki('oops.json');
var collection = lokiDatabase.addCollection('props',{
indices: ['name']
});
function hello() {
var insertjson = {name:'Ram'};
collection.insert(insertjson);
lokiDatabase.saveDatabase();
var select = collection.findOne({name:'Ram'});
console.log('select response:'+ select.name);
}
hello();
I am able to get the output with findOne method;
But here, the question is; as tutorials said, LokiJS is an in-memory database; wheras, I can see all the inserts & updates are presenting in the oops.json file.
Where we are storing/from to in-memory here?
Did I understood the concepts wrong?

enter image description here
//lokirouter.js
const db=require('./lokidb')
const router=require('express').Router
class Erouter{
static get(){
router.get("/",(req,res)=>{
db.loadDatabase({},function(){
try{
const data=db.getCollection('items').find({})
res.send(data).status(200)
}
catch(r){
res.status(500).send(`${r}`)
}
})
})
router.post("/",(req,res)=>{
db.loadDatabase({},()=>{
try{
const data=db.getCollection('items').insert(req.body.criteria)
db.saveDatabase(data)
db.save(data)
res.send(data).status(200)
}
catch(r){
res.status(500).send(`${r}`)
}
})
})
return router
}}
module.exports=Erouter
//lokidb.js
var loki=require('lokijs')
var db = new loki('loki.db');
var items = db.addCollection('items');
module.exports=db
//lokiapp.js
const lokirouter=require('./lokirouter')
const express =require("express")
const bodyParser = require('body-parser')
const app=express()
const db=require('./lokidb')
const port=8000;
app.listen(port)
console.log("Server Started"+ " "+port)
app.use(bodyParser.urlencoded({ extended: true }))
app.use(bodyParser.json())
app.use("/",lokirouter.get())

Lokijs is a document driven database,besides not necessary to store records only a json file,can also store in local database file creating local_database.db for instance.
As previous mentioned answer below you have to run it using postman.
when you insert records in request body in json format
ex:{ "criteria":{"name":"jason"} } it will get inserted into the local_database.db file.Similarly to retrive the records you can call get api. Since to find a particular record you can use findOne({name:"jason"}).

Related

What is the proper way to run fetch calls which use reactive components from a store?

I am getting two reactive variables I need from a store to use for my fetch calls. I need these fetch calls to rerun when the data in these store values change. I am able to make this work however when I reload the page it causes my app to crash because there are no values that are getting from the store. I am able to make it work if I disable ssr on the +page.js file.
I also believe it is relevant to mention that I am using a relative URL (/api) to make the fetch call because I have a proxy server to bypass CORS
What is the proper way to get this data by rerunning the fetch calls using a reactive component from a store without disabling ssr? Or is this the best/only solution?
+page.svelte
<script>
import { dateStore, shiftStore } from '../../../lib/store';
$: shift = $shiftStore
$: date = $dateStore
/**
* #type {any[]}
*/
export let comments = []
/**
* #type {any[]}
*/
let areas = []
//console.log(date)
async function getComments() {
const response = await fetch(`/api/${date.toISOString().split('T')[0]}/${shift}/1`)
comments = await response.json()
console.log(comments)
}
async function getAreas() {
const response = await fetch(`/api/api/TurnReportArea/1/${date.toISOString().split('T')[0]}/${shift}`)
areas = await response.json()
console.log(areas)
}
// both of these call function if date or shift value changes
$: date && shift && getAreas()
$: date , shift , getComments()
</script>
I tried to use the +page.js file for my fetch calls, however I cannot use the reactive values in the store in the +page.js file. Below the date variable is set as a 'Writble(Date)' When I try to add the $ in front of the value let dare = $dateStore, I get the error 'Cannot find name '$dateSrote'' If i put the $ in the fetch call I get the error 'Cannot find $date'. Even if I were able to make this work, I do not understand how my page would know to rerender if these fetch calls were ran so I do not think this is the solution. As I mentioned, the only solution I have found is to disable ssr on the +page.js, which I do not think is the best way to fix this issue.
import { dateStore, shiftStore } from "../../../lib/store"
export const load = async ({ }) => {
let shift = shiftStore
let date = dateStore
const getComments = async() => {
const commentRes = await fetch(`/api/${date.toISOString().split('T')[0]}/${shift}/1`)
const comments = await commentRes.json()
console.log(comments)
}
const getAreas = async () => {
const areasRes = await fetch(`/api/api/TurnReportArea/1/${date.toISOString().split('T')[0]}/${shift}`)
const areas = await areasRes.json()
console.log(areas)
}
return {
comments: getComments(),
areas: getAreas()
}
}

Can't access Firestore docs data after getting the doc object

I'm trying to fetch a single field value from a doc in a collection (stored in Firestore).
The following function is called (by the triggered function) to perform this query and return the result.
Firestore data structure:
After I fetch the query result into helper_token object - I cannot access the DATA (fields) in it.
I tried many things, including:
helper_token[0].device_token;
helper_token.data().device_token;
JSON.stringify(helper_token);
Nothing works for me.
The log always shows results like these:
helper_token = {}
helper_token = undefined
What am I missing? how can I get the device_token based on user?
const admin = require('firebase-admin'); //required to access the FB RT DB
admin.initializeApp(functions.config().firebase);
const db = admin.firestore();
function getHelperToken(helperId) {
//Get token from Firestore
const tokensRef = db.collection('tokens');
const helper_token = tokensRef.where('user', '==', 'TM1EOV4lYlgEIly0cnGHVmCnybT2').get();
if (helper_token.empty) {
functions.logger.log('helper_token EMPTY');
}
functions.logger.log('helper_token=' + JSON.stringify(helper_token));
return helper_token.device_token;
};
For completeness, adding here the full calling function to the above function:
//DB triggered function - upon writing a child in the HElpersInvitations reference
exports.sendHelperInvitation = functions.database.ref('/HelpersInvitations/{helper_invitation_id}')
.onCreate((snapshot, context) => {
const helperId = snapshot.val().helperId;
const title = snapshot.val().title;
const body = snapshot.val().body;
//Get the helper token by Id
functions.logger.log('HelperID=' + helperId);
functions.logger.log('getHelperToken=' + getHelperToken(helperId));
const helper_token2 = getHelperToken(helperId);
//Notification payload
const payload = {
notification: {
title: title,
body: body,
icon: 'default',
click_action: 'com.skillblaster.app.helperinvitationnotification'
}
}
// //Send the notification
functions.logger.log('helper_token [BEFORE sendToDevice]=' + helper_token2);
return admin.messaging().sendToDevice(helper_token2, payload);
});
You need to consider that the get() call is asynchornous and also that you get a list of documents and not a single doc. Can you try it with this code:
const admin = require("firebase-admin"); //required to access the FB RT DB
admin.initializeApp(functions.config().firebase);
const db = admin.firestore();
async function getHelperToken(helperId) {
//Get token from Firestore
const tokensRef = db.collection("tokens");
const helperTokens = await tokensRef
.where("user", "==", "TM1EOV4lYlgEIly0cnGHVmCnybT2")
.get();
let helper_token = "";
helperTokens.forEach((token) => {
helper_token = token.data();
});
functions.logger.log("helper_token=" + JSON.stringify(helper_token));
return helper_token.device_token;
}
As the get() call in Firestore is asynchronous you need to use an asynchronous function. You can go through this article to know more about why Firebase APIs are asynchronous. Next when we query with the where clause in Firestore we get a list of documents even if there is only one document in the list. So we have to run a for loop to get the document inside the list of documents. Now as you are returning the value from an asynchronous function the return value will be a promise in pending state. So to get the value from the promise we need to use the then() block while calling the function.
Also I think the parameter helperId you are using in the function definition is not used anywhere in the function. Though it will not make a difference I would suggest you remove it if it is not required in the function.
So consider using the following code -
const admin = require(‘firebase-admin’);
admin.initializeApp(functions.config().firebase);
const db = admin.firestore();
async function getHelperToken() {
//Get token from Firestore
const tokensRef = db.collection(‘tokens’);
const helper_token = await tokensRef.where(‘user’, ‘==’, ‘TM1EOV4lYlgEIly0cnGHVmCnybT2’).get();
let helper_token_needed;
helper_token.forEach((token) => {
helper_token_needed = token.data();
});
console.log(helper_token_needed.device_token);
return helper_token_needed.device_token;
}
//when calling to the function use then() block to get the value as a promise is returned from asynchronous function
getHelperToken().then((value)=>{console.log(value)});

Dialogflow to Firestore using Inline Fulfillment - Store all user data in one document

How do we store all user input data in one document per one chat session?
I tried this code:
'use strict';
const functions = require('firebase-functions');
const {WebhookClient} = require('dialogflow-fulfillment');
const admin = require('firebase-admin');
admin.initializeApp();
const db = admin.firestore();
process.env.DEBUG = 'dialogflow:debug';
exports.dialogflowFirebaseFulfillment = functions.https.onRequest((request, response) => {
const agent = new WebhookClient({ request, response });
function getAge(agent) {
let age = agent.parameters.age;
db.collection("users").add({age: age});
}
function getLocation(agent) {
let location = agent.parameters.location;
db.collection("users").add({location: location});
}
function getCustomerExperience(agent) {
let customerExperience = agent.query;
db.collection("users").add({customerExperience: customerExperience});
}
let intentMap = new Map();
intentMap.set('age', age);
intentMap.set('location', getLocation);
intentMap.set('customer-experience', getCustomerExperience);
agent.handleRequest(intentMap);
});
but the data were stored in different document IDs:
What I'm trying to achieve is something like this:
If I'm not being clear, please let me know. I'm new to Dialogflow, Firebase, as well as the JS language. Cheers!
You're on the right track! The fundamental problem with your original code is that collection.add() will create a new document. But you want it to create a new document sometimes, and save it in a previous document other times.
This means that, during the entire Dialogflow session, you'll need some way to know what the document name is or should be. There are a few possible ways to do this.
Use a document based on the session
Dialogflow provides a session identifier that you can get as part of the agent.session property using the dialogflow-fulfillment library, or in the session property if you're parsing the JSON request body directly.
However, this string includes forward slash / characters, which should be avoided in document names. Fortunately, the format of this string is documented to be one of the two formats:
projects/Project ID/agent/sessions/Session ID
projects/Project ID/agent/environments/Environment ID/users/User ID/sessions/Session ID
In each case, the Session ID is the last portion of this path, so you can probably use code something like this to get the ID for the session, use it as your document name, and then save an attribute (for example, age) for it:
function documentRef( agent ){
const elements = agent.session.split('/');
const lastElement = elements[elements.length - 1];
return db.collection('users').doc(lastElement);
}
async function getCourier(agent) {
const ref = documentRef( agent );
const age = agent.parameters.age;
return await ref.update({age: age});
}
Note that I have also made getCourier() an async function, because the function calls that change the database (such as ref.update()) are async functions and Dialogflow requires you to either make it an async function or explicitly return a Promise. If you wish to return a Promise instead, this would be something more like this:
function getCourier(agent) {
const ref = documentRef( agent );
const age = agent.parameters.age;
return ref.update({age: age});
}
Use the document name generated by Firestore
With this method, you'll store a document name as a Context parameter. When you go to save a value, you'll check if this document name is set. If it is, you'll do an update() using this document name. If not, you'll do an add(), get the document name, and save it in the Context parameter.
It might look something like this (untested), again for the age:
async function getCourier( agent ){
const ref = db.collection('users');
const age = agent.parameters.age;
const docContext = agent.context.get('doc');
if( !docContext ){
// We don't previously have a document, so create it
const res = await ref.add({age:age});
// And set a long-lasting context with a "name" parameter
// containing the document id
agent.context.set( 'doc', 99, {
'name': ref.id
} );
} else {
// There is a context with the document name already set
// so get the name
const docName = docContext.parameters['name'];
const docRef = ref.doc(docName);
// And save the data at this location
await docRef.update({age: age});
}
}
Again, this uses an async function. If you'd rather use a Promise, it might be something more like this:
function getCourier( agent ){
const ref = db.collection('users');
const age = agent.parameters.age;
const docContext = agent.context.get('doc');
if( !docContext ){
// We don't previously have a document, so create it
return ref.add({age:age})
.then( ref => {
// And set a long-lasting context with a "name" parameter
// containing the document id
agent.context.set( 'doc', 99, {
'name': ref.id
} );
});
} else {
// There is a context with the document name already set
// so get the name
const docName = docContext.parameters['name'];
const docRef = ref.doc(docName);
// And save the data at this location
return docRef.update({age: age});
}
}
Use a document name you've generated and saved in the context
You don't need to use the session id from the first alternative. If you have some ID or name that makes sense on your own (a username or a timestamp, for example, or some combination), then you can save this in a Context parameter and use this each time as the document name. This is a combination of the first and second approaches above (but probably simpler than the second one, since you don't need to get the document name from creating the document the fist time).

Store file in Mongo's GridFS with ExpressJS after upload

I have started building a REST api using expressJS. I am new to node so please bear with me. I want to be able to let users upload a file directly to Mongo's GridFS using a post to the /upload route.
From what I understand in expressJS documentation the req.files.image object is available in the route after uploading, which also includes a path and filename attribute. But how can I exactly read the image data and store it into GridFS?
I have looked into gridfs-stream but I can't tie ends together. Do I first need to read the file and then use that data for the writestream pipe? Or can I just use the file object from express and use those attributes to construct a writestream? Any pointers would be appreciated!
Here's a simple demo:
var express = require('express');
var fs = require('fs');
var mongo = require('mongodb');
var Grid = require('gridfs-stream');
var db = new mongo.Db('test', new mongo.Server("127.0.0.1", 27017), { safe : false });
db.open(function (err) {
if (err) {
throw err;
}
var gfs = Grid(db, mongo);
var app = express();
app.use(express.bodyParser());
app.post('/upload', function(req, res) {
var tempfile = req.files.filename.path;
var origname = req.files.filename.name;
var writestream = gfs.createWriteStream({ filename: origname });
// open a stream to the temporary file created by Express...
fs.createReadStream(tempfile)
.on('end', function() {
res.send('OK');
})
.on('error', function() {
res.send('ERR');
})
// and pipe it to gfs
.pipe(writestream);
});
app.get('/download', function(req, res) {
// TODO: set proper mime type + filename, handle errors, etc...
gfs
// create a read stream from gfs...
.createReadStream({ filename: req.param('filename') })
// and pipe it to Express' response
.pipe(res);
});
app.listen(3012);
});
I use httpie to upload a file:
http --form post localhost:3012/upload filename#~/Desktop/test.png
You can check your database if the file is uploaded:
$ mongofiles list -d test
connected to: 127.0.0.1
test.png 5520
You can also download it again:
http --download get localhost:3012/download?filename=test.png

How to use GridFS to store images using Node.js and Mongoose

I am new to Node.js. Can anyone provide me an example of how to use GridFS for storing and retrieving binary data, such as images, using Node.js and Mongoose? Do I need to directly access GridFS?
I was not satisfied with the highest rated answer here and so I'm providing a new one:
I ended up using the node module 'gridfs-stream' (great documentation there!) which can be installed via npm.
With it, and in combination with mongoose, it could look like this:
var fs = require('fs');
var mongoose = require("mongoose");
var Grid = require('gridfs-stream');
var GridFS = Grid(mongoose.connection.db, mongoose.mongo);
function putFile(path, name, callback) {
var writestream = GridFS.createWriteStream({
filename: name
});
writestream.on('close', function (file) {
callback(null, file);
});
fs.createReadStream(path).pipe(writestream);
}
Note that path is the path of the file on the local system.
As for my read function of the file, for my case I just need to stream the file to the browser (using express):
try {
var readstream = GridFS.createReadStream({_id: id});
readstream.pipe(res);
} catch (err) {
log.error(err);
return next(errors.create(404, "File not found."));
}
Answers so far are good, however, I believe it would be beneficial to document here how to do this using the official mongodb nodejs driver instead of relying on further abstractions such as "gridfs-stream".
One previous answer has indeed utilized the official mongodb driver, however they use the Gridstore API; which has since been deprecated, see here. My example will be using the new GridFSBucket API.
The question is quite broad as such my answer will be an entire nodejs program. This will include setting up the express server, mongodb driver, defining the routes and handling the GET and POST routes.
Npm Packages Used
express (nodejs web application framework to simplify this snippet)
multer (for handling multipart/form-data requests)
mongodb (official mongodb nodejs driver)
The GET photo route takes a Mongo ObjectID as a parameter to retrieve the image.
I configure multer to keep the uploaded file in memory. This means the photo file will not be written to the file system at anytime, and instead be streamed straight from memory into GridFS.
/**
* NPM Module dependencies.
*/
const express = require('express');
const photoRoute = express.Router();
const multer = require('multer');
var storage = multer.memoryStorage()
var upload = multer({ storage: storage, limits: { fields: 1, fileSize: 6000000, files: 1, parts: 2 }});
const mongodb = require('mongodb');
const MongoClient = require('mongodb').MongoClient;
const ObjectID = require('mongodb').ObjectID;
let db;
/**
* NodeJS Module dependencies.
*/
const { Readable } = require('stream');
/**
* Create Express server && Routes configuration.
*/
const app = express();
app.use('/photos', photoRoute);
/**
* Connect Mongo Driver to MongoDB.
*/
MongoClient.connect('mongodb://localhost/photoDB', (err, database) => {
if (err) {
console.log('MongoDB Connection Error. Please make sure that MongoDB is running.');
process.exit(1);
}
db = database;
});
/**
* GET photo by ID Route
*/
photoRoute.get('/:photoID', (req, res) => {
try {
var photoID = new ObjectID(req.params.photoID);
} catch(err) {
return res.status(400).json({ message: "Invalid PhotoID in URL parameter. Must be a single String of 12 bytes or a string of 24 hex characters" });
}
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'photos'
});
let downloadStream = bucket.openDownloadStream(photoID);
downloadStream.on('data', (chunk) => {
res.write(chunk);
});
downloadStream.on('error', () => {
res.sendStatus(404);
});
downloadStream.on('end', () => {
res.end();
});
});
/**
* POST photo Route
*/
photoRoute.post('/', (req, res) => {
upload.single('photo')(req, res, (err) => {
if (err) {
return res.status(400).json({ message: "Upload Request Validation Failed" });
} else if(!req.body.name) {
return res.status(400).json({ message: "No photo name in request body" });
}
let photoName = req.body.name;
// Covert buffer to Readable Stream
const readablePhotoStream = new Readable();
readablePhotoStream.push(req.file.buffer);
readablePhotoStream.push(null);
let bucket = new mongodb.GridFSBucket(db, {
bucketName: 'photos'
});
let uploadStream = bucket.openUploadStream(photoName);
let id = uploadStream.id;
readablePhotoStream.pipe(uploadStream);
uploadStream.on('error', () => {
return res.status(500).json({ message: "Error uploading file" });
});
uploadStream.on('finish', () => {
return res.status(201).json({ message: "File uploaded successfully, stored under Mongo ObjectID: " + id });
});
});
});
app.listen(3005, () => {
console.log("App listening on port 3005!");
});
I wrote a blog post on this subject; is is an elaboration of my answer. Available here
Further Reading/Inspiration:
NodeJs Streams: Everything you need to know
Multer NPM docs
Nodejs MongoDB Driver
I suggest taking a look at this question: Problem with MongoDB GridFS Saving Files with Node.JS
Copied example from the answer (credit goes to christkv):
// You can use an object id as well as filename now
var gs = new mongodb.GridStore(this.db, filename, "w", {
"chunk_size": 1024*4,
metadata: {
hashpath:gridfs_name,
hash:hash,
name: name
}
});
gs.open(function(err,store) {
// Write data and automatically close on finished write
gs.writeBuffer(data, true, function(err,chunk) {
// Each file has an md5 in the file structure
cb(err,hash,chunk);
});
});
It looks like the writeBuffer has since been deprecated.
/Users/kmandrup/private/repos/node-mongodb-native/HISTORY:
82 * Fixed dereference method on Db class to correctly dereference Db reference objects.
83 * Moved connect object onto Db class(Db.connect) as well as keeping backward compatibility.
84: * Removed writeBuffer method from gridstore, write handles switching automatically now.
85 * Changed readBuffer to read on Gridstore, Gridstore now only supports Binary Buffers no Strings anymore.
remove the fileupload library
and if it is giving some multi-part header related error than remove the content-type from the headers