I have a data, which may be simulated in the following way:
N = 10^6;%10^8;
K = 10^4;%10^6;
subs = randi([1 K],N,1);
M = [randn(N,5) subs];
M(M<-1.2) = nan;
In other words, it is a matrix, where the last row is subscripts.
Now I want to calculate nanmean() for each subscript. Also I want to save number of rows for each subscript. I have a 'dummy' code for this:
uniqueSubs = unique(M(:,6));
avM = nan(numel(uniqueSubs),6);
for iSub = 1:numel(uniqueSubs)
tmpM = M(M(:,6)==uniqueSubs(iSub),1:5);
avM(iSub,:) = [nanmean(tmpM,1) size(tmpM,1)];
end
The problem is, that it is too slow. I want it to work for N = 10^8 and K = 10^6 (see commented part in the definition of these variables.
How can I find the mean of the data in a faster way?
This sounds like a perfect job for findgroups and splitapply.
% Find groups in the final column
G = findgroups(M(:,6));
% function to apply per group
fcn = #(group) [mean(group, 1, 'omitnan'), size(group, 1)];
% Use splitapply to apply fcn to each group in M(:,1:5)
result = splitapply(fcn, M(:, 1:5), G);
% Check
assert(isequaln(result, avM));
M = sortrows(M,6); % sort the data per subscript
IDX = diff(M(:,6)); % find where the subscript changes
tmp = find(IDX);
tmp = [0 ;tmp;size(M,1)]; % add start and end of data
for iSub= 2:numel(tmp)
% Calculate the mean over just a single subscript, store in iSub-1
avM2(iSub-1,:) = [nanmean(M(tmp(iSub-1)+1:tmp(iSub),1:5),1) tmp(iSub)-tmp(iSub-1)];tmp(iSub-1)];
end
This is some 60 times faster than your original code on my computer. The speed-up mainly comes from presorting the data and then finding all locations where the subscript changes. That way you do not have to traverse the full array each time to find the correct subscripts, but rather you only check what's necessary each iteration. You thus calculate the mean over ~100 rows, instead of first having to check in 1,000,000 rows whether each row is needed that iteration or not.
Thus: in the original you check numel(uniqueSubs), 10,000 in this case, whether all N, 1,000,000 here, numbers belong to a certain category, which results in 10^12 checks. The proposed code sorts the rows (sorting is NlogN, thus 6,000,000 here), and then loop once over the full array without additional checks.
For completion, here is the original code, along with my version, and it shows the two are the same:
N = 10^6;%10^8;
K = 10^4;%10^6;
subs = randi([1 K],N,1);
M = [randn(N,5) subs];
M(M<-1.2) = nan;
uniqueSubs = unique(M(:,6));
%% zlon's original code
avM = nan(numel(uniqueSubs),7); % add the subscript for comparison later
tic
uniqueSubs = unique(M(:,6));
for iSub = 1:numel(uniqueSubs)
tmpM = M(M(:,6)==uniqueSubs(iSub),1:5);
avM(iSub,:) = [nanmean(tmpM,1) size(tmpM,1) uniqueSubs(iSub)];
end
toc
%%%%% End of zlon's code
avM = sortrows(avM,7); % Sort for comparison
%% Start of Adriaan's code
avM2 = nan(numel(uniqueSubs),6);
tic
M = sortrows(M,6);
IDX = diff(M(:,6));
tmp = find(IDX);
tmp = [0 ;tmp;size(M,1)];
for iSub = 2:numel(tmp)
avM2(iSub-1,:) = [nanmean(M(tmp(iSub-1)+1:tmp(iSub),1:5),1) tmp(iSub)-tmp(iSub-1)];
end
toc %tic/toc should not be used for accurate timing, this is just for order of magnitude
%%%% End of Adriaan's code
all(avM(:,1:6) == avM2) % Do the comparison
% End of script
% Output
Elapsed time is 58.561347 seconds.
Elapsed time is 0.843124 seconds. % ~70 times faster
ans =
1×6 logical array
1 1 1 1 1 1 % i.e. the matrices are equal to one another
Related
I wrote this matlab code in order to concatenate the results of the integration of all the columns of a matrix extracted form a multi matrix array.
"datimf" is a matrix composed by 100 matrices, each of 224*640, vertically concatenated.
In the first loop i select every single matrix.
In the second loop i integrate every single column of the selected matrix
obtaining a row of 640 elements.
The third loop must concatenate vertically all the lines previously calculated.
Anyway i got always a problem with the third loop. Where is the error?
singleframe = zeros(224,640);
int_frame_all = zeros(1,640);
conc = zeros(100,640);
for i=0:224:(22400-224)
for j = 1:640
for k = 1:100
singleframe(:,:) = datimf([i+1:(i+223)+1],:);
int_frame_all(:,j) = trapz(singleframe(:,j));
conc(:,k) = vertcat(int_frame_all);
end
end
end
An alternate way to do this without using any explicit loops (edited in response to rayryeng's comment below. It's also worth noting that using cellfun may not be more efficient than explicitly looping.):
nmats = 100;
nrows = 224;
ncols = 640;
datimf = rand(nmats*nrows, ncols);
% convert to an nmats x 1 cell array containing each matrix
cellOfMats = mat2cell(datimf, ones(1, nmats)*nrows, ncols);
% Apply trapz to the contents of each cell
cellOfIntegrals = cellfun(#trapz, cellOfMats, 'UniformOutput', false);
% concatenate the results
conc = cat(1, cellOfIntegrals{:});
Taking inspiration from user2305193's answer, here's an even better "loop-free" solution, based on reshaping the matrix and applying trapz along the appropriate dimension:
datReshaped = reshape(datimf, nrows, nmats, ncols);
solution = squeeze(trapz(datReshaped, 1));
% verify solutions are equivalent:
all(solution(:) == conc(:)) % ans = true
I think I understand what you want. The third loop is unnecessary as both the inner and outer loops are 100 elements long. Also the way you have it you are assigning singleframe lots more times than necessary since it does not depend on the inner loops j or k. You were also trying to add int_frame_all to conc before int_frame_all was finished being populated.
On top of that the j loop isn't required either since trapz can operate on the entire matrix at once anyway.
I think this is closer to what you intended:
datimf = rand(224*100,640);
singleframe = zeros(224,640);
int_frame_all = zeros(1,640);
conc = zeros(100,640);
for i=1:100
idx = (i-1)*224+1;
singleframe(:,:) = datimf(idx:idx+223,:);
% for j = 1:640
% int_frame_all(:,j) = trapz(singleframe(:,j));
% end
% The loop is uncessary as trapz can operate on the entire matrix at once.
int_frame_all = trapz(singleframe,1);
%I think this is what you really want...
conc(i,:) = int_frame_all;
end
It looks like you're processing frames in a video.
The most efficent approach in my experience would be to reshape datimf to be 3-dimensional. This can easily be achieved with the reshape command.
something along the line of vid=reshape(datimf,224,640,[]); should get you far in this regard, where the 3rd dimension is time. vid(:,:,1) then would display the first frame of the video.
Hi for my code I would like to know how to best save my variable column. column is 733x1. Ideally I would like to have
column1(y)=column, but I obtain the error:
Conversion to cell from logical is not possible.
in the inner loop. I find it difficult to access these stored values in overlap.
for i = 1:7
for y = 1:ydim % ydim = 436
%execute code %code produces different 'column' on each iteration
column1{y} = column; %'column' size 733x1 %altogether 436 sets of 'column'
end
overlap{i} = column1; %iterates 7 times.
end
Ideally I want overlap to store 7 variables saved that are (733x436).
Thanks.
I'm assuming column is calculated using a procedure where each column is dependent on the latter. If not, then there are very likely improvements that can be made to this:
column = zeros(733, 1); % Might not need this. Depends on you code.
all_columns = zeros(xdim, ydim); % Pre-allocate memory (always do this)
% Note that the first dimension is usually called x,
% and the second called y in MATLAB
overlap = cell(7, 1);
overlap(:) = {zeros(xdim, ydim)}; % Pre-allocate memory
for ii = 1:numel(overlap) % numel is better than length
for jj = 1:ydim % ii and jj are better than i and j
% several_lines_of_code_to_calculate_column
column = something;
all_columns(:, jj) = column;
end
overlap{ii} = all_columns;
end
You can access the variables in overlap like this: overlap{1}(1,1);. This will get the first element in the first cell. overlap{2} will get the entire matrix in the second cell.
You specified that you wanted 7 variables. Your code implies that you know that cells are better than assigning it to different variables (var1, var2 ...). Good! The solution with different variables is bad bad bad.
Instead of using a cell array, you could instead use a 3D-array. This might make processing later on faster, if you can vectorize stuff for instance.
This will be:
column = zeros(733, 1); % Might not need this. Depends on you code.
overlap = zeros(xdim, ydim, 7) % Pre-allocate memory for 3D-matrix
for ii = 1:7
for jj = 1:ydim
% several_lines_of_code_to_calculate_column
column = something;
all_column(:, jj, ii) = column;
end
end
I am trying to concatenate several structs. What I take from each struct depends on a function that requires a for loop. Here is my simplified array:
t = 1;
for t = 1:5 %this isn't the for loop I am asking about
a(t).data = t^2; %it just creates a simple struct with 5 data entries
end
Here I am doing concatenation manually:
A = [a(1:2).data a(1:3).data a(1:4).data a(1:5).data] %concatenation function
As you can see, the range (1:2), (1:3), (1:4), and (1:5) can be looped, which I attempt to do like this:
t = 2;
A = [for t = 2:5
a(1:t).data
end]
This results in an error "Illegal use of reserved keyword "for"."
How can I do a for loop within the concatenate function? Can I do loops within other functions in Matlab? Is there another way to do it, other than copy/pasting the line and changing 1 number manually?
You were close to getting it right! This will do what you want.
A = []; %% note: no need to initialize t, the for-loop takes care of that
for t = 2:5
A = [A a(1:t).data]
end
This seems strange though...you are concatenating the same elements over and over...in this example, you get the result:
A =
1 4 1 4 9 1 4 9 16 1 4 9 16 25
If what you really need is just the .data elements concatenated into a single array, then that is very simple:
A = [a.data]
A couple of notes about this: why are the brackets necessary? Because the expressions
a.data, a(1:t).data
don't return all the numbers in a single array, like many functions do. They return a separate answer for each element of the structure array. You can test this like so:
>> [b,c,d,e,f] = a.data
b =
1
c =
4
d =
9
e =
16
f =
25
Five different answers there. But MATLAB gives you a cheat -- the square brackets! Put an expression like a.data inside square brackets, and all of a sudden those separate answers are compressed into a single array. It's magic!
Another note: for very large arrays, the for-loop version here will be very slow. It would be better to allocate the memory for A ahead of time. In the for-loop here, MATLAB is dynamically resizing the array each time through, and that can be very slow if your for-loop has 1 million iterations. If it's less than 1000 or so, you won't notice it at all.
Finally, the reason that HBHB could not run your struct creating code at the top is that it doesn't work unless a is already defined in your workspace. If you initialize a like this:
%% t = 1; %% by the way, you don't need this, the t value is overwritten by the loop below
a = []; %% always initialize!
for t = 1:5 %this isn't the for loop I am asking about
a(t).data = t^2; %it just creates a simple struct with 5 data entries
end
then it runs for anyone the first time.
As an appendix to gariepy's answer:
The matrix concatenation
A = [A k];
as a way of appending to it is actually pretty slow. You end up reassigning N elements every time you concatenate to an N size vector. If all you're doing is adding elements to the end of it, it is better to use the following syntax
A(end+1) = k;
In MATLAB this is optimized such that on average you only need to reassign about 80% of the elements in a matrix. This might not seam much, but for 10k elements this adds up to ~ an order of magnitude of difference in time (at least for me).
Bare in mind that this works only in MATLAB 2012b and higher as described in this thead: Octave/Matlab: Adding new elements to a vector
This is the code I used. tic/toc syntax is not the most accurate method for profiling in MATLAB, but it illustrates the point.
close all; clear all; clc;
t_cnc = []; t_app = [];
N = 1000;
for n = 1:N;
% Concatenate
tic;
A = [];
for k = 1:n;
A = [A k];
end
t_cnc(end+1) = toc;
% Append
tic;
A = [];
for k = 1:n;
A(end+1) = k;
end
t_app(end+1) = toc;
end
t_cnc = t_cnc*1000; t_app = t_app*1000; % Convert to ms
% Fit a straight line on a log scale
P1 = polyfit(log(1:N),log(t_cnc),1); P_cnc = #(x) exp(P1(2)).*x.^P1(1);
P2 = polyfit(log(1:N),log(t_app),1); P_app = #(x) exp(P2(2)).*x.^P2(1);
% Plot and save
loglog(1:N,t_cnc,'.',1:N,P_cnc(1:N),'k--',...
1:N,t_app,'.',1:N,P_app(1:N),'k--');
grid on;
xlabel('log(N)');
ylabel('log(Elapsed time / ms)');
title('Concatenate vs. Append in MATLAB 2014b');
legend('A = [A k]',['O(N^{',num2str(P1(1)),'})'],...
'A(end+1) = k',['O(N^{',num2str(P2(1)),'})'],...
'Location','northwest');
saveas(gcf,'Cnc_vs_App_test.png');
here is the code I have, its not simple subtraction. We want subtract each value in one vector from each value in the other vector, within certain bounds tmin and tmax. time_a and time_b are the very long vectors with times (in ps). binsize is just for grouping times in a similar range for plotting. The longest way possible would be to loop through each element and subtract each element in the other vector, but this would take forever and we are talking about vectors with hundreds of megabytes up to gb.
function [c, dt, dtEdges] = coincidence4(time_a,time_b,tmin,tmax,binsize)
% round tmin, tmax to a intiger multiple of binsize:
if mod(tmin,binsize)~=0
tmin=tmin-mod(tmin,binsize)+binsize;
end
if mod(tmax,binsize)~=0
tmax=tmax-mod(tmax,binsize);
end
dt = tmin:binsize:tmax;
dtEdges = [dt(1)-binsize/2,dt+binsize/2];
% dtEdges = linspace((tmin-binsize/2),(tmax+binsize/2),length(dt));
c = zeros(1,length(dt));
Na = length(time_a);
Nb = length(time_b);
tic1=tic;
% tic2=tic1;
% bbMax=Nb;
bbMin=1;
for aa = 1:Na
ta = time_a(aa);
bb = bbMin;
% tic
while (bb<=Nb)
tb = time_b(bb);
d = tb - ta;
if d < tmin
bbMin = bb;
bb = bb+1;
elseif d > tmax
bb = Nb+1;
else
% tic
% [dum, dum2] = histc(d,dtEdges);
index = floor((d-dtEdges(1))/(dtEdges(end)-dtEdges(1))*(length(dtEdges)-1)+1);
% toc
% dt(dum2)
c(index)=c(index)+1;
bb = bb+1;
end
end
% if mod(aa, 200) == 0
% toc(tic2)
% tic2=tic;
% end
end
% c=c(1:end-1);
toc(tic1)
end
Well, not a final answer but a few clue to simplify and accelerate your system:
First, use cached values. For example, in your line:
index = floor((d-dtEdges(1))/(dtEdges(end)-dtEdges(1))*(length(dtEdges)-1)+1);
your loop repeat the same computations every iteration. You can calculate the value before starting the loop, cache it then reuse the stored result:
cached_dt_constant = (dtEdges(end)-dtEdges(1))*(length(dtEdges)-1) ;
Then in your loop simply use:
index = floor( (d-dtEdges(1)) / cached_dt_constant +1 ) ;
if you have so many loop iteration you'll save valuable time this way.
Second, I am not entirely sure of what the computations are trying to achieve, but you can save time again by using the indexing power of matlab. By replacing the lower part of your code like this, I get an execution time 2 to 3 time faster (and the same results obviously).
Na = length(time_a);
Nb = length(time_b);
tic1=tic;
dtEdge_span = (dtEdges(end)-dtEdges(1)) ;
cached_dt_constant = dtEdge_span * (length(dtEdges)-1) ;
for aa = 1:Na
ta = time_a(aa);
d = time_b - ta ;
iok = (d>=tmin) & (d<=tmax) ;
index = floor( (d(iok)-dtEdges(1)) ./ cached_dt_constant +1 ) ;
c(index) = c(index) +1 ;
end
toc(tic1)
end
Now there is only one loop to go through, the inner loop has been removed and replaced by vectorized calculation. By scratching the head a bit further there might be a way to do even without the top loop and use only vectorized computations. Although this will require to have enough memory to handle quite big arrays in one go.
If the precision of each value is not critical (I see you round and floor values often), try converting your initial vectors to 'single' type instead of the default matlab 'double'. that would almost double the size of array your memory will be able to handle in one go.
I have surjective functions created by matching one element in an array MatchesX.trainIdx to one or more elements in a second array MatchesX.queryIdx.
To obtain only the bijective elements of said funciton I run the same function forward
Matches1=Matcher.match(Descriptors1,Descriptors2);
and then backwards
Matches2=Matcher.match(Descriptors2,Descriptors1);
and then look for the elements occuring in both function in following fashion:
k=1;
DoubleMatches=Matches1;
for i=1:length(Matches1)
for j=1:length(Matches2)
if((Matches1(i).queryIdx==Matches2(j).trainIdx)&&(Matches1(i).trainIdx==Matches2(j).queryIdx))
DoubleMatches(k)=Matches1(i);
k=k+1;
end
end
end
DoubleMatches(k:end)=[];
This of course does the work, but it is rather unelegant and seems to bother the JIT accelerator (calc time with accel on and accel off is the same).
Can you think of a way to vectorize this expresion? Is there any other way of avoiding the JIT from "striking"?
Thanks a lot and sorry about the strange structs, I'm working with MEX-functions. Let me know if rewriting the code in "normal" arrays would help
Access to data in multi-dimensional structures is notoriously slow in MATLAB, so transforming your data to an ordinary array will certainly help:
kk = 1;
DoubleMatches = Matches1;
%// transform to regular array
Matches1queryIdx = [Matches1.queryIdx];
Matches1trainIdx = [Matches1.trainIdx];
Matches2queryIdx = [Matches2.queryIdx];
Matches2trainIdx = [Matches2.trainIdx];
%// loop through transformed data instead of structures
for ii = 1:length(Matches1queryIdx)
for jj = 1:length(Matches1queryIdx)
if((Matches1queryIdx(ii)==Matches2trainIdx(jj)) && ...
(Matches1trainIdx(ii)==Matches2queryIdx(jj)))
DoubleMatches(kk) = Matches1(ii);
kk = kk+1;
end
end
end
DoubleMatches(kk:end)=[];
There is also a solution that is almost entirely vectorized:
matches = sum(...
bsxfun(#eq, [Matches1.queryIdx], [Matches2.trainIdx].') & ...
bsxfun(#eq, [Matches1.trainIdx], [Matches2.queryIdx].'));
contents = arrayfun(#(x)..
repmat(Matches1(x),1,matches(x)), 1:numel(matches), ...
'Uniformoutput', false);
DoubleMatches2 = [contents{:}]';
Note that this can be a lot more memory intensive (it has O(N²) peak memory footprint, as opposed to O(N) for the others, although the data type at peak memory is logical and thus 8x smaller than double...). Better do some checks beforehand which one you should use.
A little test. I used the following dummy data:
Matches1 = struct(...
'queryIdx', num2cell(randi(25,1000,1)),...
'trainIdx', num2cell(randi(25,1000,1))...
);
Matches2 = struct(...
'queryIdx', num2cell(randi(25,1000,1)),...
'trainIdx', num2cell(randi(25,1000,1))...
);
and the following test:
%// Your original method
tic
kk = 1;
DoubleMatches = Matches1;
for ii = 1:length(Matches1)
for jj = 1:length(Matches2)
if((Matches1(ii).queryIdx==Matches2(jj).trainIdx) && ...
(Matches1(ii).trainIdx==Matches2(jj).queryIdx))
DoubleMatches(kk) = Matches1(ii);
kk = kk+1;
end
end
end
DoubleMatches(kk:end)=[];
toc
DoubleMatches1 = DoubleMatches;
%// Method with data transformed into regular array
tic
kk = 1;
DoubleMatches = Matches1;
Matches1queryIdx = [Matches1.queryIdx];
Matches1trainIdx = [Matches1.trainIdx];
Matches2queryIdx = [Matches2.queryIdx];
Matches2trainIdx = [Matches2.trainIdx];
for ii = 1:length(Matches1queryIdx)
for jj = 1:length(Matches1queryIdx)
if((Matches1queryIdx(ii)==Matches2trainIdx(jj)) && ...
(Matches1trainIdx(ii)==Matches2queryIdx(jj)))
DoubleMatches(kk) = Matches1(ii);
kk = kk+1;
end
end
end
DoubleMatches(kk:end)=[];
toc
DoubleMatches2 = DoubleMatches;
% // Vectorized method
tic
matches = sum(...
bsxfun(#eq, [Matches1.queryIdx], [Matches2.trainIdx].') & ...
bsxfun(#eq, [Matches1.trainIdx], [Matches2.queryIdx].'));
contents = arrayfun(#(x)repmat(Matches1(x),1,matches(x)), 1:numel(matches), 'Uniformoutput', false);
DoubleMatches3 = [contents{:}]';
toc
%// Check if all are equal
isequal(DoubleMatches1,DoubleMatches2, DoubleMatches3)
Results:
Elapsed time is 6.350679 seconds. %// ( 1×) original method
Elapsed time is 0.636479 seconds. %// (~10×) method with regular array
Elapsed time is 0.165935 seconds. %// (~40×) vectorized
ans =
1 %// indeed, outcomes are equal
Assuming Matcher.match returns array of the same objects as passed to it as arguments you can solve this like this
% m1 are all d1s which have relation to d2
m1 = Matcher.match(d1,d2);
% m2 are all d2s, which have relation to m1
% and all m1 already have backward relation
m2 = Matcher.match(d2,m1);