I'm trying to create a connection and add a document with mongoengine through an SSH tunnel.
A successful attempt with pymongo can be seen below, I simply want something similar with mongoengine. :-)
from auth import *
import pymongo
from sshtunnel import SSHTunnelForwarder
server = SSHTunnelForwarder(
(HOST_IP, HOST_PORT),
ssh_username = SSH_USER,
ssh_password = SSH_PASS,
remote_bind_address = ('localhost', 27017)
)
server.start()
client = pymongo.MongoClient('127.0.0.1', server.local_bind_port)
db = client[MONGO_DB]
db.authenticate(MONGO_USER, MONGO_PASS)
coll = db.queue_db
coll.insert({"testFile42":43})
server.stop()
mongoengine.connect(
db=DB_NAME,
host="127.0.0.1",
port=server.local_bind_port
)
Related
Files to upload will be like WFSIV0101202001.318.tar.gz,WFSIV0101202001.2624.tar.gz etc.
INPUT_FILE_PATH = 'C:\Files to upload'
try:
import os
from google.cloud import storage
import sys
import pymongo
import pymongo.errors
from pymongo import MongoClient
from pymongo.errors import ConnectionFailure
except:
print("missing modules")
try:
mongo_client = MongoClient(host="xyz.com", port=27017)
Db = mongo_client['abcd']
coll = Db['shopper_journey_sitedata']
except ConnectionFailure:
print("Connection failed")
date=[]
# Thirdpartyid=[]
input_files = os.listdir(INPUT_FILE_PATH)
# looping through input files
for input_file in input_files:
x = input_file.split(".")
date.append(x[0][5:13])
tp_site_id = x[1]
# print(tp_site_id)
cur = coll.find({"third_party_site_id":tp_site_id})
for doc in cur:
print(doc)
Now i want to parameterize the find() method for every id, so that on each iteration i should get st_site_id ?
above code i tried but ist giving error as "Datas:name error"
You can do one thing
coll.find({"third_party_site_id": { $in :
[318,2624,2621,2622,102,078]}})
If Tid is an array, then you could replace 318 in your query to Tid[I]
i've made a local server using flask and mongoDB which works great on windows, but when i moved my code to the raspberry pi, i've got an error which i couldn't figure out why it occurs.
the code im using:
1) for the flask server
from flask import Flask
from flask import jsonify
from flask import request
import pymongo
import time
import datetime
import json
app = Flask(__name__)
client = pymongo.MongoClient("localhost", 27017)
db = client['mqtt-db']
obs_collection = db['mqtt-collection']
#app.route("/obs")
def obs():
data_str = request.args.get("data")
print data_str
data = json.loads(data_str)
print data
data["date"] = datetime.datetime.now()
obs_collection.save(data)
return "success"
#app.route("/get_obs")
def get_obs():
res = []
for row in obs_collection.find():
del row['_id']
res.append(row)
return jsonify(res)
#app.route("/delete_all")
def delete_all():
res = obs_collection.delete_many({})
return jsonify({"deleted": res.deleted_count})
if __name__ == "__main__":
app.run(host="0.0.0.0", debug=True)
2) script for inserting messages into db , using mqtt protocol:
import paho.mqtt.client as mqtt
import pymongo
import json
import datetime
topic = "sensor"
host = "10.0.0.6"
client = pymongo.MongoClient("localhost", 27017)
db = client['mqtt-db']
mqtt_collection = db['mqtt-collection']
# The callback for when the client receives a CONNACK response from the server.
def on_connect(client, userdata, flags, rc):
print("Connected with result code "+str(rc))
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
client.subscribe(topic)
# The callback for when a PUBLISH message is received from the server.
def on_message(client, userdata, msg):
data_str = str(msg.payload)
data = json.loads(data_str)
print data_str
print data
data["date"] = datetime.datetime.now()
mqtt_collection.save(data)
print(msg.topic+" "+str(msg.payload))
client = mqtt.Client()
client.on_connect = on_connect
client.on_message = on_message
client.connect(host, 1883, 60)
# Blocking call that processes network traffic, dispatches callbacks and
# handles reconnecting.
# Other loop*() functions are available that give a threaded interface and a
# manual interface.
client.loop_forever()
the error occurs when i try to retrieve data from the server using "get_obs" function.
the error is: "Value Error: dictionary update sequence element #0 has length 4; 2 is required"
appreciate your help.
as #davidism suggested, the solution was to update to the latest version of Flask
I'm trying to use MongoDB on Amazon EC2 using Python3.4.3.
I followed this answer: modified security groups setting and edited "/etc/mongod.conf" (comment out bind_id). However, when I run following code, I got this error:
ServerSelectionTimeoutError: SSL handshake failed: [Errno 54] Connection reset by peer
What else should I do?
The code I run is:
import pymongo
import ssl
client = pymongo.MongoClient('ec2-**-**-*-**.us-west-2.compute.amazonaws.com', 27017,
ssl=True, ssl_keyfile='/Users/S/FILENAME.pem')
db = client["test"]
db.artist
collection = db.artist
import gzip
import json
from io import StringIO
with gzip.open('artist.json.gz', "rt") as a_file:
count=0
bulk = []
for line in a_file:
jdata = json.load(StringIO(line))
bulk.append(jdata)
count += 1
if 1000 < count:
print ('bulk insert!')
collection.insert_many(bulk)
bulk = []
count = 0
if len(bulk) > 0:
collection.insert_many(bulk)
Question: How to provide pemfile password in pymongo mongoclient in the connection string?
import pymongo
from pymongo import MongoClient
sslCAFile = data['COMMON_SETTINGS']['sslCAFile'] //reading cafile path from configurationfile
sslpemkeyfile = data['COMMON_SETTINGS']['sslpemkeyfile'] //reading pemfile path from configurationfile(which is encrypted with password)
// now i need to connect by giving the password . but i dont see any parameter for that in pymongo documentation and in authentication examples
connection =
MongoClient(mongos_ip,int(mongos_port),ssl=True,ssl_certfile=sslpemkeyfile,ssl_ca_certs=sslCAFile)
//Help me on this!!!
Unfortunately the current version of pymongo doesn't support this feature
ref: https://jira.mongodb.org/browse/PYTHON-640
What about this:
import ssl
connection = MongoClient(mongos_ip, int(mongos_port),
ssl=True,
ssl_certfile=sslpemkeyfile,
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_ca_certs=sslCAFile)
It is from here: http://api.mongodb.org/python/current/examples/authentication.html
I have a small flask application which I am deploying to Heroku.
My local configuration looks like this:
from flask import Flask
from flask.ext.mongoengine import MongoEngine
app = Flask(__name__)
app.debug = True
app.config["MONGODB_SETTINGS"] = {'DB': "my_app"}
app.config["SECRET_KEY"] = "secretpassword"
db = MongoEngine(app)
So, I know that I need to configure the app to use the Mongo URI method of connection, and I have my connection info:
mongodb://<user>:<password>#alex.mongohq.com:10043/app12345678
I am just a little stuck as to the syntax for modifying my app to connect through the URI.
So I got it working (finally):
from flask import Flask
from mongoengine import connect
app = Flask(__name__)
app.config["MONGODB_DB"] = 'app12345678'
connect(
'app12345678',
username='heroku',
password='a614e68b445d0d9d1c375740781073b4',
host='mongodb://<user>:<password>#alex.mongohq.com:10043/app12345678',
port=10043
)
Though I anticipate that various other configurations will work.
When you look at the flask-mongoengine code, you can see what configuration variables are available
So this should work:
app.config["MONGODB_HOST"] = 'alex.mongohq.com/app12345678'
app.config["MONGODB_PORT"] = 10043
app.config["MONGODB_DATABASE"] = 'dbname'
app.config["MONGODB_USERNAME"] = 'user'
app.config["MONGODB_PASSWORD"] = 'password'
db = MongoEngine(app)
I'm not sure, if app123 is the app or the database name. You might have to fiddle arround a little to get the connection. I had the same problem with Mongokit + MongoLab on Heroku :)
Also you could use the URI like this.
app.config["MONGODB_SETTINGS"] = {'DB': "my_app", "host":'mongodb://<user>:<password>#alex.mongohq.com:10043/app12345678'}
I have actually no idea, at what point "MONGODB_SETTINGS" is read, but it seemed to work, when I tried it in the shell.
I figured out how to use the flask.ext.mongoengine.MongoEngine wrapper class to do this rather than mongoengine.connect():
from flask import Flask
from flask.ext.mongoengine import MongoEngine
app = Flask(__name__)
HOST = '<hostname>' # ex: 'oceanic.mongohq.com'
db_settings = {
'MONGODB_DB': '<database>',
'MONGODB_USERNAME': '<username>',
'MONGODB_PASSWORD': '<password>',
'MONGODB_PORT': <port>,
}
app.config = dict(list(app.config.items()) + list(db_settings.items()))
app.config["MONGODB_HOST"] = ('mongodb://%(MONGODB_USERNAME)s:%(MONGODB_PASSWORD)s#'+
HOST +':%(MONGODB_PORT)s/%(MONGODB_DB)s') % db_settings
db = MongoEngine(app)
if __name__ == '__main__':
app.run()
If you're using mongohq, app.config["MONGODB_HOST"] should match the Mongo URI under Databases->Admin->Overview.
You can then follow MongoDB's tumblelog tutorial using this setup to write your first app called tumblelog.
Using python's nifty object introspection (python oh how I love you so), you can see how the MongoEngine wrapper class accomplishes this:
from flask.ext.mongoengine import MongoEngine
import inspect
print(inspect.getsource(MongoEngine))
...
conn_settings = {
'db': app.config.get('MONGODB_DB', None),
'username': app.config.get('MONGODB_USERNAME', None),
'password': app.config.get('MONGODB_PASSWORD', None),
'host': app.config.get('MONGODB_HOST', None),
'port': int(app.config.get('MONGODB_PORT', 0)) or None
}
...
self.connection = mongoengine.connect(**conn_settings)
...
self.app = app