Related
I trying to have a sha1 algorithm in Matlab.
I know I can use System.Security.Cryptography.HashAlgorithm.Create('SHA1');, but that relies on .NET, which I'd like to avoid.
I also found this thread, which suggest to use MessageDigest.getInstance("SHA-1"), but the output is not good.
I haven't found any other portable way.
Here's my code based on Wikipedia pseudo code, the result is also off. The only solution that gives result similar to online SHA1 is the .Net function.
Can somebody see the error in my function?
function [hh] = sha1(bytes_in)
% Note 1: All variables are unsigned 32-bit quantities and wrap modulo 232 when calculating, except for
% ml, the message length, which is a 64-bit quantity, and
% hh, the message digest, which is a 160-bit quantity.
% Note 2: All constants in this pseudo code are in big endian.
% Within each word, the most significant byte is stored in the leftmost byte position
%
% Initialize variables:
bytes_in = squeeze(uint8(bytes_in));
bytes_in = reshape(bytes_in, length(bytes_in), 1);
h0 = uint32(0x67452301);
h1 = uint32(0xEFCDAB89);
h2 = uint32(0x98BADCFE);
h3 = uint32(0x10325476);
h4 = uint32(0xC3D2E1F0);
% Pre-processing:
% append the bit '1' to the message e.g. by adding 0x80 if message length is a multiple of 8 bits.
% append 0 ? k < 512 bits '0', such that the resulting message length in bits
% is congruent to ?64 ? 448 (mod 512)
% append ml, the original message length in bits, as a 64-bit big-endian integer.
% Thus, the total length is a multiple of 512 bits.
%
message_len64 = uint64(length(bytes_in));
messages_len_bytes = zeros(8, 1);
for i=1:8
messages_len_bytes(i) = uint8(bitshift(message_len64, -64+i*8));
end
bytes_in = [bytes_in; 0x80];
padlen = 64-8 - mod(length(bytes_in), 64);
bytes_in = [bytes_in; zeros(padlen,1);messages_len_bytes];
assert(mod(length(bytes_in), 64) == 0);
chunk_count = length(bytes_in)/64;
% Process the message in successive 512-bit chunks:
% break message into 512-bit chunks
for i=1:chunk_count
chunk = bytes_in( ((i-1)*64+1):(i*64));
assert(length(chunk) == 64);
% Break chunk into sixteen 32-bit big-endian words w[i], 0 ? i ? 15
w = uint32(zeros(80,1));
for j=0:15
p1 = bitshift(uint32(chunk(j*4+1)),24);
p2 = bitshift(uint32(chunk(j*4+2)),16);
p3 = bitshift(uint32(chunk(j*4+3)),8);
p4 = bitshift(uint32(chunk(j*4+4)),0);
w(j+1) = p1 + p2 + p3 + p4;
end
% Message schedule: extend the sixteen 32-bit words into eighty 32-bit words:
for j=17:80
temp = bitxor(bitxor(bitxor(w(j-3),w(j-8)), w(j-14)), w(j-16));
w(j) = leftrotate32(temp, 1);
end
% Initialize hash value for this chunk:
a = h0;
b = h1;
c = h2;
d = h3;
e = h4;
for j=1:80
if j >= 1 && j <= 20
f = bitor(bitand(b,c), bitand(bitcmp(b), d));
k = 0x5A827999;
elseif j >= 21 && j <= 40
f = bitxor(bitxor(b,c), d);
k = 0x6ED9EBA1;
elseif j >= 41 && j <= 60
f = bitor(bitor(bitand(b, c),bitand(b, d)), bitand(c, d)) ;
k = 0x8F1BBCDC;
elseif j >= 61 && j <= 80
f = bitxor(bitxor(b,c), d);
k = 0xCA62C1D6;
end
temp = uint64(leftrotate32(a,5)) + uint64(f) + uint64(e) + uint64(k) + uint64(w(j));
temp = uint32(bitand(temp, uint64(0xFFFFFFFF)));
e = d;
d = c;
c = leftrotate32(b, 30);
b = a;
a = temp;
end
% Add this chunk's hash to result so far:
h0 = uint32(bitand(uint64(h0) + uint64(a), uint64(0xFFFFFFFF)));
h1 = uint32(bitand(uint64(h1) + uint64(b), uint64(0xFFFFFFFF)));
h2 = uint32(bitand(uint64(h2) + uint64(c), uint64(0xFFFFFFFF)));
h3 = uint32(bitand(uint64(h3) + uint64(d), uint64(0xFFFFFFFF)));
h4 = uint32(bitand(uint64(h4) + uint64(e), uint64(0xFFFFFFFF)));
% Produce the final hash value (big-endian) as a 160-bit number:
hh = [dec2hex(h0, 8), dec2hex(h1, 8), dec2hex(h2, 8), dec2hex(h3, 8), dec2hex(h4, 8)];
assert(length(hh) == 160/8*2)
end
end
function vout = leftrotate32(v32, v)
vout = uint32(bin2dec(circshift(dec2bin(v32, 32), -v)));
end
I have the following function:
function tx_upconv_out_transaction predict(tx_upconv_in_transaction in_trx);
tx_upconv_out_transaction predicted = tx_upconv_out_transaction::type_id::create("predicted");
//-------golden model-----
// predicted.y = (in_trx.xi * in_trx.cos - in_trx.xq * in_trx.sin)/ (2 ** 17);
$display(" xi = %d, cos = %d xq = %d sin = %d", $signed(in_trx.xi),$signed(in_trx.cos),$signed(in_trx.xq),$signed(in_trx.sin) );
predicted.y = ($signed(in_trx.xi) * $signed(in_trx.cos) - $signed(in_trx.xq) * $signed(in_trx.sin))/ (131072);
return predicted;
endfunction: predict
Where:
The field in in_trx are defined by:
bit [15:0] xi;
bit [15:0] xq;
bit [15:0] sin;
bit [15:0] cos;
For the input:
xi, qq = fffa (hex)
sin = 0
cos = 7ffe (hex)
The output (display) is:
xi = -6, cos = 32766 xq = -6 sin = 0
Where it should be:
xi = -6, cos = -2 xq = -6 sin = 0
You can declare your vectors to signed and unsigned (default). Eg.:
logic signed [3:0] signed_reg; // a 4-bit vector in range -8 to 7
From now you you will not need $signed systemcalls.
Also if you are using 16 bit 2-state variables you should consider the built in
shortint type that is a 2-state data type, 16-bit signed integer.
I have an array of samples of ECG signals 1250x1 double let us called it "a".
I need to implement 4 functions which represent features are used to characterize the signals. Energy, 4th Power,Nonlinear Energy and Curve Length
I manged to implement Energy and 4th Power
for i=1:1250
energy = sum(a.^2,i);
power4th = sum(a.^4,i);
end
Which produce 2 array (energy and power4th)
How I can produce the other 2 array? let us called them NonLE and CL.
Use vectorization instead of for loops to solve all 4 of the formulas you need
% generate some random numbers
a = rand(1000,1);
Energy = sum(a.^2);
Power4 = sum(a.^4);
NLEnergy = sum(-a(3:end).*a(1:end-2) + a(2:end).^2);
CurveLength = sum(a(2:end) - a(1:end-1));
The . operator allows element by element operations in a vector.
Actually I think you can implement your formulas without using for loop. You can use matrix multiplication characteristic. Try the code below:
len = 1250;
a = randi(10, len, 1); % // You didn' t give your vector so I generated random a..
Energy = ones(1, len) * (a.^2);
power4th = ones(1, len) * (a.^4);
NonLE = ones(1, len - 2) * ( -a(3:end) .* a(1:end-2) ) + ones(1, len - 1) * ( a(2:end).^2 );
CL = ones(1, len - 1) * ( a(2:end) - a(1:end-1) );
You don't really need a for loop for 3 of them:
energy = sum(a.^2);
power_4th = sum(a.^4);
curve_length = sum(diff(a));
For the last one, you can do something like:
nonLE = 0;
for k = 3 : length(a)
nonLE = nonLE + a(k - 1)^2 - a(k) * a(k - 2);
end
Most answers only address the already-answered question about Hamming weights but ignore the point about find and dealing with the sparsity. Apparently the answer by Shai here addresses the point about find -- but I am not yet able to verify it. My answer here does not utilise the ingenuity of other answers such as the bitshifting but good enough example answer.
Input
>> mlf=sparse([],[],[],2^31+1,1);mlf(1)=10;mlf(10)=111;mlf(77)=1010;
>> transpose(dec2bin(find(mlf)))
ans =
001
000
000
011
001
010
101
Goal
1
0
0
2
1
1
2
Fast calculation for the amount of ones in binary numbers with the sparse structure?
You can do this in tons of ways. The simplest I think would be
% Example data
F = [268469248 285213696 536904704 553649152];
% Solution 1
sum(dec2bin(F)-'0',2)
And the fastest (as found here):
% Solution 2
w = uint32(F');
p1 = uint32(1431655765);
p2 = uint32(858993459);
p3 = uint32(252645135);
p4 = uint32(16711935);
p5 = uint32(65535);
w = bitand(bitshift(w, -1), p1) + bitand(w, p1);
w = bitand(bitshift(w, -2), p2) + bitand(w, p2);
w = bitand(bitshift(w, -4), p3) + bitand(w, p3);
w = bitand(bitshift(w, -8), p4) + bitand(w, p4);
w = bitand(bitshift(w,-16), p5) + bitand(w, p5);
According to your comments, you convert a vector of numbers to binary string representations using dec2bin. Then you can achieve what you want as follows, where I'm using vector [10 11 12] as an example:
>> sum(dec2bin([10 11 12])=='1',2)
ans =
2
3
2
Or equivalently,
>> sum(dec2bin([10 11 12])-'0',2)
For speed, you could avoid dec2bin like this (uses modulo-2 operations, inspired in dec2bin code):
>> sum(rem(floor(bsxfun(#times, [10 11 12].', pow2(1-N:0))),2),2)
ans =
2
3
2
where N is the maximum number of binary digits you expect.
If you really want fast, I think a look-up-table would be handy. You can simply map, for 0..255 how many ones they have. Do this once, and then you only need to decompose an int to its bytes look the sum up in the table and add the results - no need to go to strings...
An example:
>> LUT = sum(dec2bin(0:255)-'0',2); % construct the look up table (only once)
>> ii = uint32( find( mlf ) ); % get the numbers
>> vals = LUT( mod( ii, 256 ) + 1 ) + ... % lower bytes
LUT( mod( ii/256, 256 ) + 1 ) + ...
LUT( mod( ii/65536, 256 ) + 1 ) + ...
LUT( mod( ii/16777216, 256 ) + 1 );
Using typecast (as suggested by Amro):
>> vals = sum( reshape(LUT(double(typecast(ii,'uint8'))+1), 4, [] ), 1 )';
Run time comparison
>> ii = uint32(randi(intmax('uint32'),100000,1));
>> tic; vals1 = sum( reshape(LUT(typecast(ii,'uint8')+1), 4, [] ), 1 )'; toc, %//'
>> tic; vals2 = sum(dec2bin(ii)-'0',2); toc
>> dii = double(ii); % type issues
>> tic; vals3 = sum(rem(floor(bsxfun(#times, dii, pow2(1-32:0))),2),2); toc
Results:
Elapsed time is 0.006144 seconds. <-- this answer
Elapsed time is 0.120216 seconds. <-- using dec2bin
Elapsed time is 0.118009 seconds. <-- using rem and bsxfun
Here is an example to show #Shai's idea of using a lookup table:
% build lookup table for 8-bit integers
lut = sum(dec2bin(0:255)-'0', 2);
% get indices
idx = find(mlf);
% break indices into 8-bit integers and apply LUT
nbits = lut(double(typecast(uint32(idx),'uint8')) + 1);
% sum number of bits in each
s = sum(reshape(nbits,4,[]))
you might have to switch to uint64 instead if you have really large sparse arrays with large indices outside the 32-bit range..
EDIT:
Here is another solution for you using Java:
idx = find(mlf);
s = arrayfun(#java.lang.Integer.bitCount, idx);
EDIT#2:
Here is yet another solution implemented as C++ MEX function. It relies on std::bitset::count:
bitset_count.cpp
#include "mex.h"
#include <bitset>
void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
{
// validate input/output arguments
if (nrhs != 1) {
mexErrMsgTxt("One input argument required.");
}
if (!mxIsUint32(prhs[0]) || mxIsComplex(prhs[0]) || mxIsSparse(prhs[0])) {
mexErrMsgTxt("Input must be a 32-bit integer dense matrix.");
}
if (nlhs > 1) {
mexErrMsgTxt("Too many output arguments.");
}
// create output array
mwSize N = mxGetNumberOfElements(prhs[0]);
plhs[0] = mxCreateDoubleMatrix(N, 1, mxREAL);
// get pointers to data
double *counts = mxGetPr(plhs[0]);
uint32_T *idx = reinterpret_cast<uint32_T*>(mxGetData(prhs[0]));
// count bits set for each 32-bit integer number
for(mwSize i=0; i<N; i++) {
std::bitset<32> bs(idx[i]);
counts[i] = bs.count();
}
}
Compile the above function as mex -largeArrayDims bitset_count.cpp, then run it as usual:
idx = find(mlf);
s = bitset_count(uint32(idx))
I decided to compare all the solutions mentioned so far:
function [t,v] = testBitsetCount()
% random data (uint32 vector)
x = randi(intmax('uint32'), [1e5,1], 'uint32');
% build lookup table (done once)
LUT = sum(dec2bin(0:255,8)-'0', 2);
% functions to compare
f = {
#() bit_twiddling(x) % bit twiddling method
#() lookup_table(x,LUT); % lookup table method
#() bitset_count(x); % MEX-function (std::bitset::count)
#() dec_to_bin(x); % dec2bin
#() java_bitcount(x); % Java Integer.bitCount
};
% compare timings and check results are valid
t = cellfun(#timeit, f, 'UniformOutput',true);
v = cellfun(#feval, f, 'UniformOutput',false);
assert(isequal(v{:}));
end
function s = lookup_table(x,LUT)
s = sum(reshape(LUT(double(typecast(x,'uint8'))+1),4,[]))';
end
function s = dec_to_bin(x)
s = sum(dec2bin(x,32)-'0', 2);
end
function s = java_bitcount(x)
s = arrayfun(#java.lang.Integer.bitCount, x);
end
function s = bit_twiddling(x)
p1 = uint32(1431655765);
p2 = uint32(858993459);
p3 = uint32(252645135);
p4 = uint32(16711935);
p5 = uint32(65535);
s = x;
s = bitand(bitshift(s, -1), p1) + bitand(s, p1);
s = bitand(bitshift(s, -2), p2) + bitand(s, p2);
s = bitand(bitshift(s, -4), p3) + bitand(s, p3);
s = bitand(bitshift(s, -8), p4) + bitand(s, p4);
s = bitand(bitshift(s,-16), p5) + bitand(s, p5);
end
The times elapsed in seconds:
t =
0.0009 % bit twiddling method
0.0087 % lookup table method
0.0134 % C++ std::bitset::count
0.1946 % MATLAB dec2bin
0.2343 % Java Integer.bitCount
This gives you the rowsums of the binary numbers from the sparse structure.
>> mlf=sparse([],[],[],2^31+1,1);mlf(1)=10;mlf(10)=111;mlf(77)=1010;
>> transpose(dec2bin(find(mlf)))
ans =
001
000
000
011
001
010
101
>> sum(ismember(transpose(dec2bin(find(mlf))),'1'),2)
ans =
1
0
0
2
1
1
2
Hope someone able to find faster rowsummation!
Mex it!
Save this code as countTransBits.cpp:
#include "mex.h"
void mexFunction( int nout, mxArray* pout[], int nin, mxArray* pin[] ) {
mxAssert( nin == 1 && mxIsSparse(pin[0]) && mxGetN( pin[0] ) == 1,
"expecting single sparse column vector input" );
mxAssert( nout == 1, "expecting single output" );
// set output, assuming 32 bits, set to 64 if needed
pout[0] = mxCreateNumericMatrix( 32, 1, mxUINT32_CLASS, mxREAL );
unsigned int* counter = (unsigned int*)mxGetData( pout[0] );
for ( int i = 0; i < 32; i++ ) {
counter[i] = 0;
}
// start working
mwIndex *pIr = mxGetIr( pin[0] );
mwIndex* pJc = mxGetJc( pin[0] );
double* pr = mxGetPr( pin[0] );
for ( mwSize i = pJc[0]; i < pJc[1]; i++ ) {
if ( pr[i] != 0 ) {// make sure entry is non-zero
unsigned int entry = pIr[i] + 1; // cast to unsigned int and add 1 for 1-based indexing in Matlab
int bit = 0;
while ( entry != 0 && bit < 32 ) {
counter[bit] += ( entry & 0x1 ); // count the lsb
bit++;
entry >>= 1; // shift right
}
}
}
}
Compile it in Matlab
>> mex -largeArrayDims -O countTransBits.cpp
Run the code
>> countTransBits( mlf )
Note that the output count in 32 bins lsb to msb.
The bitcount FEX contribution offers a solution based on the lookup table approach, but is better optimized. It runs more than twice as fast as the bit twiddling method (i.e. the fastest pure-MATLAB method reported by Amro) over a 1 million uint32 vector, using R2015a on my old laptop.
What are good hashing functions (fast, good distribution, few collisions) for hashing 2d and 3d vectors composed of IEEE 32bit floats. I assume general 3d vectors, but algorithms assuming normals (always in [-1,1]) are also welcome. I also do not fear bit-manipulation as IEEE floats are alsways IEEE floats.
Another more general problem is hashing an Nd float-vector, where N is quite small (3-12) and constant but not known at compile time. At the moment I just take these floats as uints and XOR them together, which is probably not the best solution.
There's a spatial hash function described in Optimized Spatial Hashing for Collision Detection of Deformable Objects. They use the hash function
hash(x,y,z) = ( x p1 xor y p2 xor z
p3) mod n
where p1, p2, p3 are large
prime numbers, in our case 73856093,
19349663, 83492791, respectively. The
value n is the hash table size.
In the paper, x, y, and z are the discretized coordinates; you could probably also use the binary values of your floats.
I have two suggestions.
Assume a grid cell of size l and quantize the x, y and z co-ordinates by computing ix = floor(x/l), iy = floor(y/l), and iz = floor(z/l), where ix, iy and iz are integers. Now use the hash function defined in Optimized Spatial Hashing for Collision Detection of Deformable Objects
If you don't do the quantization, it wont be sensitive to closeness(locality).
Locality Sensitive Hashing has been mentioned for hashing higher dimensional vectors. Why not use them for 3d or 2d vectors as well? A variant of LSH using adapted for Eucledian distance metric (which is what we need for 2d and 3d vectors) is called Locality Sensitive Hashing using p-stable distributions. A very readable tutorial is here.
I wrote this in Python based on the comments seen here,
l = 5
n = 5
p1,p2,p3 = 73856093, 19349663, 83492791
x1 = [33,4,11]
x2 = [31,1,14]
x3 = [10,44,19]
def spatial_hash(x):
ix,iy,iz = np.floor(x[0]/l), np.floor(x[1]/l), np.floor(x[2]/l)
return (int(ix*p1) ^ int(iy*p2) ^ int(iz*p3)) % n
print (spatial_hash(x1))
print (spatial_hash(x2))
print (spatial_hash(x3))
It gives
1
1
3
It seemed to work.
In C++
#include <cstdlib>
#include <iostream>
#include <unordered_map>
#include <vector>
#include <random>
#include <eigen3/Eigen/Dense>
using namespace Eigen;
using namespace std;
const int HASH_SIZE = 200;
//const float MAX = 500.0;
const float L = 0.2f;
const float mmin = -1.f;
const float mmax = 1.f;
unordered_map<int, vector<Vector3d>> map ;
inline size_t hasha(Vector3d &p) {
int ix = (unsigned int)((p[0]+2.f) / L);
int iy = (unsigned int)((p[1]+2.f) / L);
int iz = (unsigned int)((p[2]+2.f) / L);
return (unsigned int)((ix * 73856093) ^ (iy * 19349663) ^ (iz * 83492791)) % HASH_SIZE;
}
int main(int argc, char** argv) {
std::default_random_engine generator;
std::uniform_real_distribution<double> distribution(-1.0,1.0);
for(size_t i=0;i<300;i++){
float x = distribution(generator);
float y = distribution(generator);
float z = distribution(generator);
Vector3d v(x,y,z);
std::cout << hasha(v) << " " << v[0] << " " << v[1] << " " << v[2] << std::endl;
map[hasha(v)].push_back(v);
vector<Vector3d> entry = map[hasha(v)];
std::cout << "size " << entry.size() << std::endl;
}
for (const auto & [ key, value ] : map) {
cout << key << std::endl;
vector<Vector3d> v = map[key];
float average = 0.0f;
for (int i=0; i<v.size(); i++){
for (int j=0; j<v.size(); j++){
if (i!=j){
Vector3d v1 = v[i];
Vector3d v2 = v[j];
std::cout << " dist " << (v1-v2).norm() << std::endl;
}
}
}
}
}