MongoDB aggregate framework very slow when have lookup - mongodb

this is my method, when the server runs this mothod, it will take 18s.
but when i run the same query, use NoSQLBoost, it just takes 5s.
public Integer queryUserEduTotal(UserAdminV1StepListRequest body) throws ParseException {
Criteria criteria = Criteria.where(Column.SYSTEM_STATUS).is(true);
criteria.and(UserColumn.APP_ID).is(body.getAppId());
criteria.and(UserColumn.STATUS).is(2);
Criteria criteria1 = Criteria.where(UserColumn.USER_TYPE).exists(false);
Criteria criteria2 = Criteria.where(UserColumn.USER_TYPE).is(UserTypeEnum.USER_REGISTER.getKey());
Criteria criteria3 = new Criteria();
criteria3.orOperator(criteria1, criteria2);
criteria.andOperator(criteria3, Criteria.where(UserColumn.SYSTEM_CREATE_TIME).gte(sdf.parse(body.getStart())), Criteria.where(UserColumn.SYSTEM_CREATE_TIME).lte(sdf.parse(body.getEnd())));
LookupOperation lookup = lookup("user_edu_item_info", "_id", UserEduItemColumn.USER_ID, "edu_info");
Criteria criteria4 = Criteria.where(Column.SYSTEM_STATUS).is(true);
criteria4.and(UserColumn.APP_ID).is(body.getAppId());
criteria4.and("edu_info._id").exists(true);
Aggregation typedAggregation1 = Aggregation.newAggregation(
match(criteria),
lookup,
match(criteria4),
Aggregation.count().as(Constant.COUNT)
);
AggregationResults<BasicDBObject> aggregationResults = mongoTemplate.aggregate(typedAggregation1, User.class, BasicDBObject.class);
int count = 0;
if (aggregationResults.getMappedResults().size() == 0) {
count = 0;
} else {
String document = JSON.toJSONString(aggregationResults.getMappedResults().get(0));
JSONObject result = JSON.parseObject(document);
count = result.getInteger(Constant.COUNT);
}
return count;
}
and, i run the same code on the local by using the java program.it takes 5s too.
i dont know why the server takes so much time.
MongoDB configuration:
MongoClientOptions.Builder builder = new xxx()
builder.connectionsPerHost(100);
builder.minConnectionsPerHost(10);
builder.connectTimeout(30000);
builder.threadsAllowedToBlockForConnectionMultiplier(10);
builder.serverSelectionTimeout(30000);
builder.socketTimeout(0);
builder.maxWaitTime(120000);
builder.heartbeatConnectTimeout(30000);
builder.heartbeatSocketTimeout(30000);
builder.heartbeatFrequency(10000);
builder.minHeartbeatFrequency(500);
builder.localThreshold(15);
this is my local java code
public class Test {
public static void main(String[] args) {
// Link code omitted
MongoTemplate mongoTemplate = new MongoTemplate(mongoDbFactory);
search(mongoTemplate);
try {
mongoDbFactory.destroy();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void search(MongoTemplate mongoTemplate){
Criteria criteria = Criteria.where("systemStatus").is(true);
//criteria code omitted, sane whit the server code
Aggregation typedAggregation1 = Aggregation.newAggregation(
match(criteria),
lookup,
match(criteria4),
Aggregation.count().as("count")
);
AggregationResults<BasicDBObject> aggregationResults = mongoTemplate.aggregate(typedAggregation1, User.class, BasicDBObject.class);
}
}

Related

MongoDB multiple document updation passing multiple parameters

We have a single document updation,
public void Update<T>(string collectionName, T value,string feildName,int id)
{
try
{
mongoServer.Connect();
MongoCollection<T> mongoCollection = mongoDatabase.GetCollection<T>(collectionName);
IMongoQuery Marker = Query.EQ(feildName, id);
BsonDocument doc = value.ToBsonDocument();
mongoCollection.update(doc);
}
catch
{
throw;
}
finally
{
mongoServer.Disconnect();
}
}

How to get CURRENT version label number through DFC code?

public static IDfCollection getVersions() {
IDfQuery query = clientX.getQuery();// obtain an idfObject
query.setDQL(
"select r_object_id,object_name,r_version_label,owner_name from dm_document(all) where i_chronicle_id='090008868006d5be'");
IDfCollection collection = null;
try {
collection = query.execute(SessionFile.getSession(), IDfQuery.DF_READ_QUERY);
} catch (DfException e) {
e.printStackTrace();
}
return collection;
}
public class Versions {
public static void main(String[] args) {
IDfCollection collection = AllOperations.getVersions();
try {
int versionsCount = collection.getValueCount("r_object_id");
//IDfSysObject dfSysObject=(IDfSysObject) SessionFile.getSession().getObject(new DfId("09000886800a143e"));
while (collection.next()) {
//String versionNum = dfSysObject.getVersionLabels().getImplicitVersionLabel();
int i = 0;
while (i < versionsCount) {
System.out.println(collection.getRepeatingValue("r_version_label", i));
i++;
}
}
} catch (DfException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
Output of the above code is:
1.0,
1.1,
1.2,
CURRENT
in place of "CURRENT" label, I want to "get version number" like 2.0 or 3.0 etc which is latest document version number.
needed output like below :
First of all, this line:
int versionsCount = collection.getValueCount("r_object_id");
should rather be
int versionsCount = collection.getValueCount("r_version_label");
and has to be in the while(collection.next()) loop, this way it will properly iterate over all r_version_label values, so the code will now output:
1.0
1.1
1.2
CURRENT
2.0
you can now get rid of CURRENT (or in general get rid of all non values which can't be parsed to a float number).
IDfDocument doc = (IDfDocument) session.getObject(colDocuments.getId("r_object_id"));
String versionLabel = doc.getAllRepeatingStrings("r_version_label", null);

org.hibernate.search.bridge.BridgeException: Exception while calling bridge#objectToString

I am able to insert record and able index them but i am facing an exception while searching
org.hibernate.search.bridge.BridgeException: Exception while calling bridge#objectToString
class: com.edoors.formBean.Hib_cons_Cv
path: cons_cv
I am able search on all coulmn of table except blob column
Field Bridge
public class ByteArrayBridge implements TwoWayStringBridge {
public String objectToString(Object object) {
byte[] data = (byte[]) object;
StringWriter writer = new StringWriter();
InputStream is = null;
try {
is = new ByteArrayInputStream(data);
new AutoDetectParser().parse(is,new WriteOutContentHandler(writer),new Metadata(),new
ParseContext());
return is.toString();
} catch (Exception e) {
System.out.println("Exception "+e);
}
return writer.toString();
}
public Object stringToObject(String string) {
byte[] data=string.getBytes();
Object obj=data;
return obj;
}
}
DAO Class ::
public List searchConsultantByTitle(String jobtitle)
{
List list=null;
Session session = hiberUtil.openSession();
Transaction tx = null;
try{
tx = session.beginTransaction();
FullTextSession fullTextSession = Search.getFullTextSession(session);
QueryBuilder queryBuilder =
fullTextSession.getSearchFactory().buildQueryBuilder().forEntity(Hib_cons_Cv.class).get();
org.apache.lucene.search.Query luceneQuery = null;
luceneQuery =
queryBuilder.keyword().fuzzy().withThreshold(0.7f).onField("cons_cv").matching(jobtitle).createQuery();
FullTextQuery hibernateQuery = fullTextSession.createFullTextQuery(luceneQuery, Hib_cons_Cv.class);
int resultSize = hibernateQuery.getResultSize();
System.out.println(".....resultSize..............................."+resultSize);
}
catch(Exception e)
{
System.out.println(e.getMessage());
}
}
POJO Class
#Entity
#AnalyzerDef(name = "customanalyzer", tokenizer = #TokenizerDef(factory =
KeywordTokenizerFactory.class), filters = {
#TokenFilterDef(factory = LowerCaseFilterFactory.class),
#TokenFilterDef(factory = SnowballPorterFilterFactory.class, params = {
#Parameter(name = "language", value = "English") }) })
#Indexed
public class Hib_cons_Cv {
#Column(name = "cons_cv", unique = false, nullable = false, length = 59296)
#Lob
#Field(analyze = Analyze.NO, store = Store.YES)
#FieldBridge(impl = ByteArrayBridge.class)
private Blob cons_cv;
//setters and getters
}
I also got this error (with no stack trace). Turned out i put in the wrong field name. It was actually using a field with no bridging.

The column name ... was not found in this ResultSet

We are using java jdk 1.7.0_45, postgresql jdbc connector postgresql-9.3-1100.jdbc41.jar.
Here is a synopsis of our problem, as much as possible of code pasted below.
This code:
ResultSet rs = DbConn.getInstance().doQuery("Select d.deptId from Depts d");
while (rs.next()){
System.out.println(rs.getInt("d.deptId"));
Produces the error:
org.postgresql.util.PSQLException: The column name d.deptId was not found in this ResultSet.
This code:
ResultSet rs = DbConn.getInstance().doQuery("Select d.deptId from Depts d");
while (rs.next()){
System.out.println(rs.getInt("deptId"));
Produces no error.
Is there a way, besides removing the "d." from the first query, to make the first code snippet not throw the error message?
Here is the source code:
public class JoinTest {
#Test
public void test(){
boolean pass = false;
try {
ResultSet rs = DbConn.getInstance().doQuery("Select d.deptId from Depts d");
String label = rs.getMetaData().getColumnLabel(1); // What do you get?
System.out.println("label = " + label);
while (rs.next()){
System.out.println(rs.getInt("d.deptId"));
pass = true;
}
} catch (SQLException e) {
e.printStackTrace();
pass=false;
}
assertTrue(pass);
}
#Test
public void test2(){
boolean pass = false;
try {
ResultSet rs = DbConn.getInstance().doQuery("Select d.deptId from Depts d");
while (rs.next()){
System.out.println(rs.getInt("deptId"));
pass = true;
}
} catch (SQLException e) {
e.printStackTrace();
pass=false;
}
assertTrue(pass);
}
}
public class DbConn {
private static String url = "jdbc:postgresql://server:port/schema";
private static Properties props = new Properties(); {
props.setProperty("user","userid");
props.setProperty("password","passwprd");
}
private Connection conn;
private DbConn(){}
private static DbConn instance;
public static DbConn getInstance() throws SQLException{
if (instance == null){
instance = new DbConn();
instance.conn = DriverManager.getConnection(url, props);
}
return instance;
}
public ResultSet doQuery(String query) throws SQLException{
Logger.log("DbConn.doQuery: " + query);
Statement st = conn.createStatement();
ResultSet rs = st.executeQuery(query);
return rs;
}
}
}
The query:
select d.deptId from Depts d
produces a single-column resultset with the result-alias "deptId". There is no "d.deptId" column. If you want one, you can request that as the column alias instead:
select d.deptId AS "d.deptId" from Depts d
PgJDBC can't do anything about this because it has no idea that the resultset column "deptId" is related to the "d.deptId" in the select-list. Teaching it about that would force it to understand way more about the SQL it processes than would be desirable, and lead to maintenance and performance challenges.
The second one works - why isn't that acceptable?
You can also do this:
System.out.println(rs.getInt(1));
If you change the query you have to change the code, too.

How to implement boolean retrieval using hitcollector in below scenario

I am running my code on TREC documents and right now implementing scoring scheme to get number of relevant documents. However now i want to implement boolean retrieval, I am trying to use HitCollector.
below is my code..
public class BatchSearch {
private BatchSearch() {}
/** Simple command-line based search demo. */
public static void main(String[] args) throws Exception {
String usage =
"Usage:\tjava BatchSearch [-index dir] [-simfn similarity] [-field f] [-queries file]";
if (args.length > 0 && ("-h".equals(args[0]) || "-help".equals(args[0]))) {
System.out.println(usage);
System.out.println("Supported similarity functions:\ndefault: DefaultSimilary (tfidf)\n");
System.exit(0);
}
String index = "index";
String field = "contents";
String queries = null;
String simstring = "default";
for(int i = 0;i < args.length;i++) {
if ("-index".equals(args[i])) {
index = args[i+1];
i++;
} else if ("-field".equals(args[i])) {
field = args[i+1];
i++;
} else if ("-queries".equals(args[i])) {
queries = args[i+1];
i++;
} else if ("-simfn".equals(args[i])) {
simstring = args[i+1];
i++;
}
}
Similarity simfn = null;
if ("default".equals(simstring)) {
simfn = new DefaultSimilarity();
} else if ("bm25".equals(simstring)) {
simfn = new BM25Similarity();
} else if ("dfr".equals(simstring)) {
simfn = new DFRSimilarity(new BasicModelP(), new AfterEffectL(), new NormalizationH2());
} else if ("lm".equals(simstring)) {
simfn = new LMDirichletSimilarity();
}
if (simfn == null) {
System.out.println(usage);
System.out.println("Supported similarity functions:\ndefault: DefaultSimilary (tfidf)");
System.out.println("bm25: BM25Similarity (standard parameters)");
System.out.println("dfr: Divergence from Randomness model (PL2 variant)");
System.out.println("lm: Language model, Dirichlet smoothing");
System.exit(0);
}
IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(index)));
IndexSearcher searcher = new IndexSearcher(reader);
searcher.setSimilarity(simfn);
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_41);
BufferedReader in = null;
if (queries != null) {
in = new BufferedReader(new InputStreamReader(new FileInputStream(queries), "UTF-8"));
} else {
in = new BufferedReader(new InputStreamReader(new FileInputStream("queries"), "UTF-8"));
}
QueryParser parser = new QueryParser(Version.LUCENE_41, field, analyzer);
while (true) {
String line = in.readLine();
if (line == null || line.length() == -1) {
break;
}
line = line.trim();
if (line.length() == 0) {
break;
}
String[] pair = line.split(" ", 2);
Query query = parser.parse(pair[1]);
doBatchSearch(in, searcher, pair[0], query, simstring);
}
reader.close();
}
/**
* This function performs a top-1000 search for the query as a basic TREC run.
*/
public static void doBatchSearch(BufferedReader in, IndexSearcher searcher, String qid, Query query, String runtag)
throws IOException {
// Collect enough docs to show 5 pages
TopDocs results = searcher.search(query, 1000);
ScoreDoc[] hits = results.scoreDocs;
HashMap<String, String> seen = new HashMap<String, String>(1000);
int numTotalHits = results.totalHits;
int start = 0;
int end = Math.min(numTotalHits, 1000);
for (int i = start; i < end; i++) {
Document doc = searcher.doc(hits[i].doc);
String docno = doc.get("docno");
// There are duplicate document numbers in the FR collection, so only output a given
// docno once.
if (seen.containsKey(docno)) {
continue;
}
seen.put(docno, docno);
System.out.println(qid+" Q0 "+docno+" "+i+" "+hits[i].score+" "+runtag);
}
}
}
The scoring is done in doBatchSearch and now i want to implement HitCollector here.