I have data collection in MongoDB.
My data was built in model tree and relationship between child node and parent node is property parentid.
Here is my data architecture
And here is my sample data
{"_id":{"$oid":"5ebd05b52f3700008500220b"},"username":"DHBK","password":"123456","lastname":"DHBK","useremail":"dhbk#edu.com.vn","usertel":"0907111001","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":null,"comid":"DHBK","comdepartment":"DHBK","usercode":"DHBK_0001","usertype":"ADMIN_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500220c"},"username":"KHOA_DIEN","password":"123456","lastname":"KHOA_DIEN","useremail":"KHOA_DIEN#edu.com.vn","usertel":"0907111002","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"DHBK","comid":"DHBK","comdepartment":"KHOA_DIEN","usercode":"DHBK_0002","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500220d"},"username":"KHOA_XD","password":"123456","lastname":"KHOA_XD","useremail":"KHOA_XD#edu.com.vn","usertel":"0907111003","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"DHBK","comid":"DHBK","comdepartment":"KHOA_XD","usercode":"DHBK_0003","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500220e"},"username":"KHOA_CNTT","password":"123456","lastname":"KHOA_CNTT","useremail":"KHOA_CNTT#edu.com.vn","usertel":"0907111004","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"DHBK","comid":"DHBK","comdepartment":"KHOA_CNTT","usercode":"DHBK_0004","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500220f"},"username":"BOMON_TUDONG","password":"123456","lastname":"BOMON_TUDONG","useremail":"BOMON_TUDONG#edu.com.vn","usertel":"0907111005","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_DIEN","comid":"DHBK","comdepartment":"KHOA_DIEN","usercode":"DHBK_0005","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002210"},"username":"BOMON_VIENTHONG","password":"123456","lastname":"BOMON_VIENTHONG","useremail":"BOMON_VIENTHONG#edu.com.vn","usertel":"0907111006","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_DIEN","comid":"DHBK","comdepartment":"KHOA_DIEN","usercode":"DHBK_0006","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002211"},"username":"BOMON_HETHONG","password":"123456","lastname":"BOMON_HETHONG","useremail":"BOMON_HETHONG#edu.com.vn","usertel":"0907111007","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_DIEN","comid":"DHBK","comdepartment":"KHOA_DIEN","usercode":"DHBK_0007","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002212"},"username":"BOMON1_XD","password":"123456","lastname":"BOMON1_XD","useremail":"BOMON1_XD#edu.com.vn","usertel":"0907111008","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_XD","comid":"DHBK","comdepartment":"KHOA_XD","usercode":"DHBK_0008","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002213"},"username":"BOMON2_XD","password":"123456","lastname":"BOMON2_XD","useremail":"BOMON2_XD#edu.com.vn","usertel":"0907111009","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_XD","comid":"DHBK","comdepartment":"KHOA_XD","usercode":"DHBK_0009","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002214"},"username":"BOMON3_XD","password":"123456","lastname":"BOMON3_XD","useremail":"BOMON3_XD#edu.com.vn","usertel":"0907111010","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_XD","comid":"DHBK","comdepartment":"KHOA_XD","usercode":"DHBK_0010","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002215"},"username":"TRUONGKHOA_BMVT","password":"123456","lastname":"TRUONGKHOA_BMVT","useremail":"TRUONGKHOA_BMVT#edu.com.vn","usertel":"0907111011","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"BOMON_VIENTHONG","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0011","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002216"},"username":"PHOKHOA_BMVT","password":"123456","lastname":"PHOKHOA_BMVT","useremail":"PHOKHOA_BMVT#edu.com.vn","usertel":"0907111012","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"BOMON_VIENTHONG","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0012","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002217"},"username":"THUKY_BMVT","password":"123456","lastname":"THUKY_BMVT","useremail":"THUKY_BMVT#edu.com.vn","usertel":"0907111013","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"BOMON_VIENTHONG","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0013","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002218"},"username":"GV_BMVT","password":"123456","lastname":"GV_BMVT","useremail":"GV_BMVT#edu.com.vn","usertel":"0907111014","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"BOMON_VIENTHONG","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0014","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f37000085002219"},"username":"SV1_BMVT","password":"123456","lastname":"SV1_BMVT","useremail":"SV1_BMVT#edu.com.vn","usertel":"0907111015","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"GV_BMVT","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0015","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500221a"},"username":"SV2_BMVT","password":"123456","lastname":"SV2_BMVT","useremail":"SV2_BMVT#edu.com.vn","usertel":"0907111016","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"GV_BMVT","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0016","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500221b"},"username":"SV3_BMVT","password":"123456","lastname":"SV3_BMVT","useremail":"SV3_BMVT#edu.com.vn","usertel":"0907111017","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"GV_BMVT","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0017","usertype":"USER_COM"}
{"_id":{"$oid":"5ebd05b52f3700008500221c"},"username":"SV4_BMVT","password":"123456","lastname":"SV4_BMVT","useremail":"SV4_BMVT#edu.com.vn","usertel":"0907111018","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"GV_BMVT","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0018","usertype":"USER_COM"}
{"_id":{"$oid":"5ec642b2412b0000e70021a5"},"username":"KHOA_KT","password":"123456","lastname":"KHOA_KT","useremail":"KHOA_KT#edu.com.vn","usertel":"0907111002","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"DHKT","comid":"DHBK","comdepartment":"KHOA_KT","usercode":"DHKT_0002","usertype":"USER_COM"}
{"_id":{"$oid":"5ec642b2412b0000e70021a8"},"username":"BOMON_KTDOANHNGHIEP","password":"123456","lastname":"BOMON_KTDOANHNGHIEP","useremail":"BOMON_KTDOANHNGHIEP#edu.com.vn","usertel":"0907111005","userdate":"2020-05-05","userstatus":"ACTIVE","userparentid":"KHOA_KT","comid":"DHBK","comdepartment":"KHOA_KT","usercode":"DHKT_0005","usertype":"USER_COM"}
{"_id":{"$oid":"5ece3517b8d5570916d013f6"},"username":"SV5_BMVT","password":"123","lastname":"SV5_BMVT","useremail":"SV5_BMVT#edu.com.vn","usertel":"0907111019","userdate":"2020-05-14","userstatus":"ACTIVE","userparentid":"GV_BMVT","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0019","usertype":"USER_COM"}
{"_id":{"$oid":"5eddf0a9b8d5570916dae6ff"},"username":"SV6_BMVT","password":"123456","lastname":"SV6_BMVT","useremail":"SV6_BMVT#edu.com.vn","usertel":"0907111020","userdate":"2020-06-08","userstatus":"ACTIVE","userparentid":"GV_BMVT","comid":"DHBK","comdepartment":"BOMON_VIENTHONG","usercode":"DHBK_0019","usertype":"USER_COM"}
Now I want to get all child node of specific parent node. For example, I want to get all child node of DHBK node.
I have completed MongoDB shell to query this requirement.
Here is my MongoDB shell
var descendants=[]
var stack=[];
var item = db.users.findOne({username:"DHBK"});
stack.push(item);
while (stack.length>0){
var currentnode = stack.pop();
var children = db.users.find({userparentid:currentnode.username});
while(true === children.hasNext()) {
var child = children.next();
descendants.push(child.username);
stack.push(child);
}
}
descendants.join(",")
It worked and showed me correct result. Here is my output result
KHOA_DIEN,KHOA_XD,KHOA_CNTT,BOMON1_XD,BOMON2_XD,BOMON3_XD,BOMON_TUDONG,BOMON_VIENTHONG,BOMON_HETHONG,TRUONGKHOA_BMVT,PHOKHOA_BMVT,THUKY_BMVT,GV_BMVT,SV1_BMVT,SV2_BMVT,SV3_BMVT,SV4_BMVT,SV5_BMVT,SV6_BMVT
Then I write Go code to implement this MongoDB shell.
Here is my code
package main
import (
"context"
"fmt"
"strings"
"time"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func main() {
GetAllChildOfNode("DHBK")
}
func GetAllChildOfNode(node string) error {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
client, err := mongo.Connect(ctx, options.Client().ApplyURI("URI string"))
if err != nil {
return err
}
defer client.Disconnect(ctx)
database := client.Database("users")
users := database.Collection("users")
var descendants []string
var stack []string
err = users.FindOne(ctx, bson.M{"username": "DHBK"}).Decode(&stack)
leng := len(stack)
for leng > 0 {
//I HaVE TROUBLE HERE
currentnode := stack.
}
return nil
}
But I have trouble implementing push, pop method and while loop as MongoDB shell by use Go.
Take a look if you are okay with $graphLookup limitations.
If so, you can do it dbside in a single aggregation query:
db.collection_name.aggregate([
{$match: {username: "DHBK"}}, // select root document
{
$graphLookup: { // load the tree as embedded docs
from: "collection_name",
startWith: "$username",
connectFromField: "username",
connectToField: "userparentid",
as: "descendants",
}
},
{$project: { // add root element if needed
descendants: {$concatArrays: [["$$ROOT"], "$descendants"] }
}},
{$unwind: "$descendants"}, // final formatting steps
{$replaceRoot: {newRoot: "$descendants"}},
{$project: {descendants:0}}
])
I'm having trouble finding objects in a nested array. I need to find home/away within league array which has an events array.
Example JSON:
{
"sportId":4,
"last":266178326,
"league":[
{
"id":423,
"name":"Germany - Bundesliga",
"events":[
{
"id":1125584543,
"starts":"2020-06-07T17:00:00Z",
"home":"SC Rasta Vechta",
"away":"EnBW Ludwigsburg",
"rotNum":"2601",
"liveStatus":0,
"status":"I",
"parlayRestriction":0,
"altTeaser":false,
"resultingUnit":"Regular"
},
{
"id":1125585441,
"starts":"2020-06-10T18:30:00Z",
"home":"Ratiopharm Ulm",
"away":"Crailsheim Merlins",
"rotNum":"2617",
"liveStatus":0,
"status":"I",
"parlayRestriction":0,
"altTeaser":false,
"resultingUnit":"Regular"
}
]
},
{
"id":268,
"name":"ABA - Adriatic League",
"events":[
{
"id":1122419811,
"starts":"2020-05-07T19:34:00Z",
"home":"Test 1(Do Not Wager)",
"away":"Test 2(Do Not Wager)",
"rotNum":"999998",
"liveStatus":0,
"status":"I",
"parlayRestriction":1,
"altTeaser":false,
"resultingUnit":"Regular"
}
]
},
{
"id":487,
"name":"NBA",
"events":[
{
"id":1120192519,
"starts":"2020-05-01T17:00:00Z",
"home":"Test Team B",
"away":"Test Team A",
"rotNum":"123",
"liveStatus":0,
"status":"O",
"parlayRestriction":0,
"altTeaser":false,
"resultingUnit":"Regular"
}
]
}
]
}
For example finding the league name "Germany - Bundesliga" I solved it by doing
// retrieve league by searching in the fixture collection
func FindLeagueFixture(name string) (pinnacle.Fixtures, pinnacle.League, error) {
var fixtures []pinnacle.Fixtures
err := db.C(FIXTURES).Find(
bson.M{"league.name": bson.RegEx{
Pattern: name,
Options: "i",
}}).All(&fixtures)
if err != nil {
return pinnacle.Fixtures{}, pinnacle.League{}, err
}
But now I have to event home/away names within league events. For example, finding "SC Rasta Vechta". What's the best way to handle this?
I've tried something like (No regex usage yet, since I'm having trouble already. Only trying count, not doing the whole unmarshaling for now)
// retrieve sport team by searching in the fixture collection
func FindHomeOrAwayFixture(name string) (pinnacle.Fixtures, pinnacle.League, error) {
var fixtures []pinnacle.Fixtures
// find home
c, err := db.C(FIXTURES).Find(
bson.M{"league": bson.M{"$elemMatch": bson.M{"home": name}}}).Count()
if err != nil {
return pinnacle.Fixtures{}, pinnacle.League{}, err
}
fmt.Println(c)
}
I want to get rows where:
{repair field has "ac" OR {repair is "tv" and phone field in range 1091-1100}}
I am trying the following query:
type M map[string]interface{}
conditions := M{"name": M{"$regex": "me"},
"$or": []M{M{"repair": M{"$eq": "ac"}},
"$and": []M{M{"repair": M{"$eq": "tv"}}, M{"phone": M{"$gte": 1091, "$lte": 1100}}}}}
fmt.Println(conditions)
err = c.Find(conditions).Sort("phone").Limit(20).All(&j)
However, I am getting a compile error:
index must be non-negative integer constant
cannot use []M literal (type []M) as type M in array or slice literal.
You're missing one M{ before "$and" and after you add that don't forget to add another closing brace }.
the good and the bad for comparison.
I don't know what driver your using, but I would probably do something like this..
package main
import (
"log"
"time"
mgo "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
const (
databaseString = ""
)
var db DataStore
type DataStore struct {
Session *mgo.Session
}
// database
func (ds *DataStore) ConnectToDB() {
mongoDBDialInfo := &mgo.DialInfo{
Addrs: []string{"localhost:27017"},
Timeout: 1 * time.Hour,
}
sess, err := mgo.DialWithInfo(mongoDBDialInfo)
if err != nil {
sess.Refresh()
panic(err)
}
sess.SetMode(mgo.Monotonic, true)
db.Session = sess
}
// J is the expected mongo return object
type J struct {
ID bson.ObjectId `bson:"_id,omitempty" json:"_id"`
// example data below
Status string `bson:"status" json:"status"`
}
func init() {
db.ConnectToDB()
}
func main() {
colectionString := ""
// probably best if this was a mapped mgo struct see above
// var j bson.M
var j J
// your orignal code
// I don't know why you are using $eq couldn't you just do bson.M{"repair":"ac"}, and bson.M{"repair":"tv"}
conditions := bson.M{"name": bson.M{"$regex": "me"},
"$or": []bson.M{
bson.M{"repair": bson.M{"$eq": "ac"}},
},
"$and": []bson.M{
bson.M{"repair": bson.M{"$eq": "tv"}},
bson.M{"phone": bson.M{"$gte": 1091, "$lte": 1100}},
}}
err := db.Session.DB(databaseString).C(colectionString).Find(conditions).Sort("phone").Limit(20).All(&j)
if err != nil {
log.Fatal(err)
}
}
I would probably also end up creating a separate package for the mongo connection stuff so that I can write wrapper functions around calls,
func FindItem(db *mgo.Session, id string) (data, error) {
defer sess.Close()
var res data //some data type struct
err := sess.DB("my db").C("my collection").Find(bson.M{"user": someID}).One(&data)
return data, err
}
then I'd be able to do things like this, which allow for concurrency
res, err := packagemain.FindItem(sess.Copy(), someID)
and your original code your you where missing } and ,. I suggest you use go vet or an ide which vets the code for you. Also, mgo is the mongo driver you probably what you want to be using if you're not using it already.
My data layer uses Mongo aggregation a decent amount, and on average, queries are taking 500-650ms to return. I am using mgo.
A sample query function is shown below which represents what most of my queries look like.
func (r userRepo) GetUserByID(id string) (User, error) {
info, err := db.Info()
if err != nil {
log.Fatal(err)
}
session, err := mgo.Dial(info.ConnectionString())
if err != nil {
log.Fatal(err)
}
defer session.Close()
var user User
c := session.DB(info.Db()).C("users")
o1 := bson.M{"$match": bson.M{"_id": id}}
o2 := bson.M{"$project": bson.M{
"first": "$first",
"last": "$last",
"email": "$email",
"fb_id": "$fb_id",
"groups": "$groups",
"fulfillments": "$fulfillments",
"denied_requests": "$denied_requests",
"invites": "$invites",
"requests": bson.M{
"$filter": bson.M{
"input": "$requests",
"as": "item",
"cond": bson.M{
"$eq": []interface{}{"$$item.active", true},
},
},
},
}}
pipeline := []bson.M{o1, o2}
err = c.Pipe(pipeline).One(&user)
if err != nil {
return user, err
}
return user, nil
}
The user struct I have looks like the following..
type User struct {
ID string `json:"id" bson:"_id,omitempty"`
First string `json:"first" bson:"first"`
Last string `json:"last" bson:"last"`
Email string `json:"email" bson:"email"`
FacebookID string `json:"facebook_id" bson:"fb_id,omitempty"`
Groups []UserGroup `json:"groups" bson:"groups"`
Requests []Request `json:"requests" bson:"requests"`
Fulfillments []Fulfillment `json:"fulfillments" bson:"fulfillments"`
Invites []GroupInvite `json:"invites" bson:"invites"`
DeniedRequests []string `json:"denied_requests" bson:"denied_requests"`
}
Based on what I have provided, is there anything obvious that would suggest why my queries are averaging 500-650ms?
I know that I am probably swallowing a bit of a performance hit by using aggregation pipeline, but I wouldn't expect it to be this bad.
.. is there anything obvious that would suggest why my queriers are averaging 500-650ms?
Yes, there is. You are calling mgo.Dial() before executing each query. mgo.Dial() has to connect to the MongoDB server every time, which you close right after the query. The connection may very likely take hundreds of milliseconds to estabilish, including authentication, allocating resources (both at server and client side), etc. This is very wasteful.
This method is generally called just once for a given cluster. Further sessions to the same cluster are then established using the New or Copy methods on the obtained session. This will make them share the underlying cluster, and manage the pool of connections appropriately.
Create a global session variable, connect on startup once (using e.g. a package init() function), and use that session (or a copy / clone of it, obtained by Session.Copy() or Session.Clone()).
For example:
var session *mgo.Session
var info *db.Inf // Use your type here
func init() {
var err error
if info, err = db.Info(); err != nil {
log.Fatal(err)
}
if session, err = mgo.Dial(info.ConnectionString()); err != nil {
log.Fatal(err)
}
}
func (r userRepo) GetUserByID(id string) (User, error) {
sess := session.Clone()
defer sess.Close()
// Now we use sess to execute the query:
var user User
c := sess.DB(info.Db()).C("users")
// Rest of the method is unchanged...
}