OrientDB1.0.1 - Unable to process 34 million records - orientdb

Question

We are using OrientDB 1.0.1 and I need to process a class/table of 34 million records. On my machine with 8 GB memory, I am getting "Exception in thread "main" java.lang.OutOfMemoryError: GC overhead limit exceeded" error after inserting about 1 million records. 96% of my physical memory is occupied. Increasing JVM heap size to 8 GB, helped in processing 2 million records but it failed at 2 millions. I tried on a physical machine with 16 GB and this problem occurred after processing 3.8 million records. I need to process all 34 million records and find unique ids. The problem seems to be clearly with orientdb's buffers getting filled.
int skipRecordCount=0;
String queryStr = "select id from Table1 WHERE id is not null SKIP +Integer.toString(skipRecordCount)+" LIMIT 10000";
Set uniqueIdsSet = new TreeSet();
List idsResult = odb.db.query(new OSQLSynchQuery(queryStr));
while (!idsResult.isEmpty())
{
for (ODocument id: idsResult)
{
uniqueIdsSet.add(id.field("id").toString());
}
skipRecordCount+=10000;
queryStr = "select id from Table1 WHERE id is not null SKIP "+Integer.toString(skipRecordCount)+" LIMIT 10000";
idsResult = odb.db.query(new OSQLSynchQuery(idsQueryStr));
}

Answer 1

I've created a testcase for 2.2.20
I've added an index hash notunique, without null, and successfully executed all the test with 2 million records, random ids, in 3 minutes, using in memory testdb, on a 16gb system (osx), with 12gb heap max (not needed!), and process size was 4.9gb
When I changed to 34 millions the insert phase of testcase, for random ids, was completed in 33 minutes, with a process size of 7.2gb (3gb direct), then created index in 15 minutes, with a process size of 8.2gb (4gb direct), and completed quickly the testcase with add of unique ids in TreeSet
Used
"CREATE INDEX test.id NOTUNIQUE_HASH_INDEX METADATA {ignoreNullValues : true}"
and
"SELECT key FROM index:test.id WHERE key NOT IN [NULL] SKIP "+Integer.toString(skipRecordCount)+" LIMIT 10000"
import com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx;
import com.orientechnologies.orient.core.metadata.schema.OClass;
import com.orientechnologies.orient.core.metadata.schema.OSchema;
import com.orientechnologies.orient.core.metadata.schema.OType;
import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.Random;
public class SelectUniqueIDs {
public static void createTreeSet (ODatabaseDocumentTx db) {
int skipRecordCount=0;
String queryStr = "select id from test WHERE id is not null SKIP "+Integer.toString(skipRecordCount)+" LIMIT 10000";
Set uniqueIdsSet = new TreeSet();
List<ODocument> idsResult = db.query(new OSQLSynchQuery(queryStr));
while (!idsResult.isEmpty())
{
for (ODocument id: idsResult)
{
uniqueIdsSet.add(id.field("id").toString());
}
skipRecordCount+=10000;
// queryStr = "SELECT id FROM test WHERE id IS NOT NULL SKIP "+Integer.toString(skipRecordCount)+" LIMIT 10000";
// Using above query index on test.id is not used as we see from message, starting from skip 50000
// INFO: $ANSI{green {db=test}} [TIP] Query 'SELECT id FROM test WHERE id IS NOT NULL SKIP 50000 LIMIT 10000' fetched more than 50000 records: to speed up the execution, create an index or change the query to use an existent index
queryStr = "SELECT key FROM index:test.id WHERE key NOT IN [NULL] SKIP "+Integer.toString(skipRecordCount)+" LIMIT 10000";
idsResult = db.query(new OSQLSynchQuery(queryStr));
}
System.out.println("**** Inserted "+uniqueIdsSet.size()+" ids in uniqueIdsSet TreeSet ****");
}
public static final void main(String[] args) {
int i;
long maxMemory = Runtime.getRuntime().maxMemory();
long totalMemory = Runtime.getRuntime().totalMemory();
long freeMemory = Runtime.getRuntime().freeMemory();
System.out.println("**** Initial Heap maxMemory="+maxMemory+" , totalMemory="+totalMemory+" , freeMemory="+freeMemory+" ****");
final ODatabaseDocumentTx db = new ODatabaseDocumentTx("memory:testdb").create();
final OSchema schema = db.getMetadata().getSchema();
final OClass clazz = schema.createClass("test");
clazz.createProperty("id", OType.DOUBLE);
for (i=0 ; i < 34000000; i++){
int r =(int)(Math.random() * 100000 + 1);
db.command(new OCommandSQL("INSERT INTO test(id) VALUES ("+r+")")).execute();
}
db.command(new OCommandSQL("CREATE INDEX test.id NOTUNIQUE_HASH_INDEX METADATA {ignoreNullValues : true}")).execute();
maxMemory = Runtime.getRuntime().maxMemory();
totalMemory = Runtime.getRuntime().totalMemory();
long insertMemory = Runtime.getRuntime().freeMemory();
System.out.println("**** Inserted "+i+" ids; Heap maxMemory="+maxMemory+" , totalMemory="+totalMemory+" , freeMemory="+insertMemory+" ****");
createTreeSet(db);
final List<ODocument> count = db.query(new OSQLSynchQuery("SELECT count(*) as ids FROM test"));
Long ids = (Long) count.get(0).field("ids");
maxMemory = Runtime.getRuntime().maxMemory();
totalMemory = Runtime.getRuntime().totalMemory();
long countMemory = Runtime.getRuntime().freeMemory();
System.out.println("**** Counted "+ids+" ids; Heap maxMemory="+maxMemory+" , totalMemory="+totalMemory+" , freeMemory="+countMemory+" ****");
final List<ODocument> docs = db.query(new OSQLSynchQuery("SELECT FROM test LIMIT 100"));
for (i=0 ; i < 10; i++){
Double value = (Double) docs.get(i).field("id");
System.out.print(i+"="+value+" ");
}
System.out.println();
maxMemory = Runtime.getRuntime().maxMemory();
totalMemory = Runtime.getRuntime().totalMemory();
long selectMemory = Runtime.getRuntime().freeMemory();
System.out.println("**** Selected "+i+" ids; Heap maxMemory="+maxMemory+" , totalMemory="+totalMemory+" , freeMemory="+selectMemory+" ****");
}
}