Pandas groupby apply with multiindex performance issues - categories

Question

import pandas as pd
import numpy as np
from collections import OrderedDict
import gc
import datetime
df = pd.DataFrame(np.random.rand(10000000, 5), columns=['A', 'B', 'C', 'D', 'E'])
df['A'] = 0.25*((df['A']/0.25).astype(int))
df['B'] = 0.25*((df['B']/0.25).astype(int))
df['A']=df['A'].astype(str)
df['B']=df['B'].astype(str)
df['ix1'] = df['A']
df['ix2'] = df['B']
df['A1']=df['A'].astype('category')
df['B1']=df['B'].astype('category')
gc.collect()
Question 1:
groupby and apply functions takes longer than dataframe.count, doing effective the same thing. how do i optimize here?
this takes ~17 seconds
df.groupby(['A', 'B']).apply(genSummary)
this takes only 3 seconds
df.groupby(['A', 'B']).count()
Question 2:
I need to apply custom functions to the groupby objects. After set multiindex, group by string is slower than without index...
def genSummary(group):
return pd.Series(OrderedDict([('Counts', np.count_nonzero(group['C'])),
('Sum', np.sum(group['D'])),
('Wavg', np.ma.average(group['E'], weights=group['C'])),
('Wavg', np.ma.average(group['E']*(group['C']>0.5), weights=group['C']*(group['C']>0))),
])
)
1 group by string takes ~8.1 seconds
df.groupby(['A', 'B']).apply(genSummary)
2 group by category takes ~6.3 seconds
df.groupby(['A1', 'B1']).apply(genSummary)
df.sort_values(['ix1', 'ix2'], inplace=True)
gc.collect()
df.set_index(['ix1', 'ix2'], inplace=True)
gc.collect()
3 set multiindex, group by string takes ~7.2 seconds
df.groupby(['A', 'B']).apply(genSummary)
4 set multiindex, group by index takes ~5. seconds
df.groupby(level=[0,1]).apply(genSummary)
5 set multiindex, group by category takes ~4.6 seconds
df.groupby(['A1', 'B1']).apply(genSummary)
df = df.reset_index()
gc.collect()
6 reset_index, group by category takes ~4.9 seconds
df.groupby(['A1', 'B1']).apply(genSummary)
7 reset_index, group by string takes ~6.2 seconds
df.groupby(['A', 'B']).apply(genSummary)