I made a code to compare 2 times . The structure of the code constitutes of a main body getting the 2 times. A class and 2 methods are converting the times into second just to make an integer. The problem is that the variable showing the integer doesn't pass to the main body. The code is as follow.
class Time():
def __init__(self, other=None):
self.other = other
def comparison(self, other):
self.other = other
return other > self
def time_to_int(self, other):
self.other = other
other = self.hour * 3600 + self.minute * 60 + self.second
print( other )
start = Time()
start.hour = 2.0
start.minute = 87
start.second = 98
start_time = Time()
start.time_to_int( start_time )
end = Time()
end.hour = 3.0
end.minute = 87
end.second = 98
end_time = Time()
end.time_to_int( end_time )
print( start_time, end_time )
start_time.comparison( end_time )
The result is
12518.0
16118.0
<__main__.Time object at 0x7f9ca1854110> <__main__.Time object at 0x7f9ca18541d0>
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-1-dc5298ddf4b1> in <module>()
31
32
---> 33 start_time.comparison(end_time)
34
<ipython-input-1-dc5298ddf4b1> in comparison(self, other)
5 def comparison(self, other):
6 self.other=other
----> 7 return other > self
8 def time_to_int(self, other):
9 self.other=other
TypeError: '>' not supported between instances of 'Time' and 'Time'
Your comparison method uses other > self, and the TypeError tells you that this is not defined. What's your question?
I'm trying to get this code to run but I get this error above. Can someone please help? I've tried reading about this in other posts but I don't really know how to apply it here. I'm trying to iterate over lines of this database and select 1400 random ones. It blocks on the error above.
def paragraph_generator(test=True, itersize=5000, year=None, state=None):
con, cur = database_connection.connect(cursor_type="server")
cur.itersize = itersize
while True:
sql = f"""
SELECT
text_id,
lccn_sn,
date,
ed,
seq,
chroniclingamerica_meta.statefp,
chroniclingamerica_meta.countyfp,
text_ocr
FROM
chroniclingamerica natural join chroniclingamerica_meta
WHERE date_part('year',date) BETWEEN 1860 AND 1920
ORDER BY RANDOM()
LIMIT 1400
"""
if test:
sql = (
sql + " limit 10000"
) # limit 1000 means it only goes through 1000 lines of the database
else:
pass
print(sql)
cur.execute(sql)
for p in cur.fetchall():
tokens = stem_text(p[-1]) # Stem
# print(tokens)
tokens = (
p[-1]
.translate(str.maketrans("", "", punct))
.replace("\n", " ")
.lower()
.split(" ")
)
tokens_3 = [
a for a in tokens if len(a) == 3 if a in wn_lemmas
] # For 3-letter words, only keep WordNet recognized tokens
tokens = gensim.parsing.preprocessing.remove_short_tokens(
tokens, minsize=4
) # Remove 1-, 2-, and 3-letter words
tokens = tokens + tokens_3 # Add back in 3-letter WordNet-recognized tokens
tokens = gensim.parsing.preprocessing.remove_stopword_tokens(
tokens, stopwords=stop_words
) # Remove stopwords in stopword list above
print("THIS IS THE LENGTH OF TOKENS")
a = len(tokens)
print(a)
if len(tokens) != 0:
ocr_2 = 1 - (
len([a for a in tokens if a in wn_lemmas]) / len(tokens)
) # Generate a measure for proportion of OCR errors in a page
else:
ocr_2 = float("nan")
print("THIS IS OCR")
print(ocr_2)
ocr = ocr_2
if ocr < 0.75 and ~np.isnan(
ocr
): # If the % of OCR in a page is less than 75%, then keep the page and all tokens
tokens = tokens
else:
tokens = [] # Otherwise, give it an empty list (i.e. drop the page)
yield tokens
con.close()
Error:
cur.execute(sql)
psycopg2.ProgrammingError: can't call .execute() on named cursors more than once
Im trying to follow an exercise on calculating the maximum drawdown and maximum drawdown duration of a market market neutral vs a long-only trading strategy.
I followed the code to the T and has worked perfectly up until now, and I seem to be getting a ValueError Exception. What code do I need to change for my code to work?
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from MaxDD_Function import calculateMaxDD
# CALCUALTING MAXDD AND CREATING THE FUNCTION.
def calculateMaxDD(cumret):
highwatermark = np.zeros(cumret.shape)
drawdown = np.zeros(cumret.shape)
drawdownduration = np.zeros(cumret.shape)
for t in np.arange(1, cumret.shape[0]):
highwatermark[t] = (np.maximum(highwatermark[t -1]), cumret[t])
drawdown[t] = ((1+ cumret[t] )/(1 + highwatermark[t]) - 1)
if drawdown[t] == 0:
drawdownduration[t] == 0
else:
drawdownduration[t] = drawdownduration[t -1] + 1
maxDD, i = np.min(drawdown, np.argmin(drawdown)) # drawdown < 0 always
maxDDD = np.max(drawdownduration)
return (maxDD, maxDDD, i)
# First part of example. Read the csv data and calculate.
#The first dataframe/set for my strategy
df = pd.read_csv('IGE_daily.csv')
#print (df.head())
df.sort_values(by= 'Date', inplace = True)
dailyret = df.loc[:, 'Adj Close'].pct_change()
excessRet = ((dailyret - 0.04)/252)
sharpeRatio = ((np.sqrt(252)*np.mean(excessRet))/np.std(excessRet))
print (sharpeRatio)
#Second part of example
#This is the second dataframe/set for my strategy.
df2 = pd.read_csv('SPY.csv')
#The new data frame, with both datasets.
df = pd.merge (df, df2, on = 'Date', suffixes = ('_IGE', '_SPY'))
df['Date'] = pd.to_datetime(df['Date'])
df.set_index('Date', inplace = True)
df.sort_index(inplace = True)
dailyret = df [['Adj Close_IGE', 'Adj Close_SPY' ]].pct_change() # Daily
Returns
dailyret.rename(columns = {"Adj Close_IGE": "IGE", "Adj Close_SPY": "SPY"
}, inplace = True)
netRet = (dailyret['IGE'] - dailyret['SPY'])/2
sharpeRatio = np.sqrt(252) * np.mean(netRet)/np.std(netRet)
print (sharpeRatio)
cumret = np.cumprod(1 + netRet) - 1 #Cumalative return
#print (plt.plot(cumret))
#print (plt.show()) # Remember to always run plt.show to see the plot in
terminal.
maxDrawdown, maxDrawdownDuration, startDrawdownDay =
calculateMaxDD(cumret.values)
maxDrawdown = calculateMaxDD(cumret.values)
print (maxDrawdown)
Here are the results I got from my above mentioned code:
Ivies-MacBook-Pro:Quant_Trading Ivieidahosa$ python Ex3_4.py
-46.10531783058014
0.7743286831426566
Traceback (most recent call last):
File "Ex3_4.py", line 76, in <module>
maxDrawdown = calculateMaxDD(cumret.values)
File "Ex3_4.py", line 15, in calculateMaxDD
highwatermark[t] = (np.maximum(highwatermark[t -1]), cumret[t])
ValueError: invalid number of arguments
I expected the output on themaxDrawdown to be -0.09529268047208683,maxDrawdwnduration to be 497 andstartDrawdownday to be 1223.
Q: What code do I need to change for my code to work?
Your code uses a call to a numpy function having a defined a minimum-call-signature as: np.maximum( <array_like_A>, <array_like_B> )
This simply fails to meet the expected behaviour once only one of the expected pair of values was delivered in the reported line ( see the closing parenthesis ), or a scalar or any other, non-array-like type of object(s) were attempted to be delivered into the call-signature:
highwatermark[t] = ( np.maximum( highwatermark[t-1] ), cumret[t] )
where a tuple was attempted to get constructed on the right hand side of the value-assignment (well, actually an object-reference gets assigned in python, sure, but was trying to remain short here to tell that fast for an easy reading ), the first item of which was expected to get assigned to a returned value from a call to the above documented np.maximum(...) function. And Hic Sunt Leones ...
May like to start further bug-tracing with a cross-check-ing of the state of objects and the call-signature:
try:
for t in np.arange( 1, cumret.shape[0] ):
print( "The shape of <highwatermark[t-1]>-object was: ",
highwatermark[t-1].shape, " for t == ", t
)
except:
print( "The <highwatermark[t-1]>-object was not a numpy array",
" for t == ", t
)
finally:
print( np.maximum.__doc__ )
I'm implementing a fail safe handover procedure in ROS and I'm using python scripts to do so.
I'm using the optical sensor from a mouse to keep under control the acceleration of the object so I can detect when is falling. Everything seems to works fine but now I want to give give a limit to the monitoring procedure (let's say 1000 times) before declaring the handover succeded. The problem is that the function read that I use for the mouse get stucked, if no movement are detected the next iteration is not performed. How can I read from the device without encountering this issue?
Here is the code I'm using to read from the mouse:
def getMouseEvent():
buf = file.read(3)
x, y = struct.unpack( "bb", buf[1:] ) # <--- X and Y deltas.
return [x , y]
Here the loop I want to implement
release_grasp()
rospy.loginfo( "Force detected -- Release mode active")
# If the object is falling regrasp it.
detected= False
trials = 0
while (not(detected) and trials < 1000):
trials = trials + 1
rospy.loginfo ("Acc monitored for the" + str(trials) + "th time"
if fall_test():
cilindrical_grasp()
rospy.loginfo("Fall detected -- Object regrasped")
detected = True
rate.sleep()
The output I get blocks to a given iteration until the mouse does not detect some kind of movement.
UPDATE: Here is the full code
#!/usr/bin/env python2
import rospy
import numpy
import struct
from reflex_sf_msgs.msg import SFPose
from matteo.msg import force
from matteo.msg import acc
# Defining force treshold in each direction ( to be completed and tuned )
rospy.init_node('DetectionFail')
xt = 0.5
yt = xt
zt = 0.3
# For the future try to handle the initialization.
fx = None
fy = None
fz = None
ax = None
ay = None
rate = rospy.Rate(100) # <--- Rate Hz
#-----------------------------MOUSE-----------------------------------#
# Open the mouse device. To be sure if it is "mouse2" type in the terminal: cat /proc/bus/input/devices, look for the device whose name is "Logitech optical USB mouse" and get the name of the handler. If you need root permissions type: sudo chmod 777 /dev/input/(handler)
file = open ("/dev/input/mouse3" , "rb")
#Defining the function to read mouse deltas.
def getMouseEvent():
buf = file.read(3);
x,y = struct.unpack( "bb", buf[1:] ); # <--- X and Y deltas.
return [x , y]
#Defining the function to estimate the acceleraton.
def acc_comp():
vx_old = 0
vy_old = 0
vx_new = getMouseEvent()[0]
vy_new = getMouseEvent()[1]
x_acc = (vx_old - vx_new)*100
y_acc = (vy_old - vy_new)*100
vx_old = vx_new
vy_old = vy_new
return [x_acc , y_acc]
#---------------------------------------------------------------------#
#Defining function fall test
def fall_test():
if ( acc_comp()[1] >= 3000 or acc_comp()[1] <= -3000 ):
return True
else:
return False
#---------------------------------------------------------------------#
# Initialize hand publisher.
hand_pub = rospy.Publisher('/reflex_sf/command', SFPose, queue_size=1)
rospy.sleep(0.5)
#---------------------------------------------------------------------#
# Defining sferical grasp.
def cilindrical_grasp():
hand_pub.publish ( 2.5 , 2.5 , 2.5, 0)
#---------------------------------------------------------------------#
# Define release position.
def release_grasp():
hand_pub.publish ( 2, 2 , 2 , 0)
#---------------------------------------------------------------------#
# Define test for the force measure
def force_treshold ( fx, fy , fz):
if ( fx > xt and fy > yt or fz > zt):
return True
else:
return False
#---------------------------------------------------------------------#
# Callback function to save the datas obtained by the force sensor
def callback_force(msg):
global fx
global fy
global fz
fx = msg.fx
fy = msg.fy
fz = msg.fz
# Main loop.
def main():
#Apply the sferical grasp.
rospy.loginfo("Applying grasp")
cilindrical_grasp()
while not(rospy.is_shutdown()):
rospy.Subscriber("/Forces", force, callback_force )
if force_treshold ( fx , fy , fz ):
release_grasp()
rospy.loginfo( "Force detected -- Release mode active")
# If the object is falling regrasp it.
detected= False
trials = 0
while (not(detected) and trials < 1000):
trials = trials +1
if fall_test():
cilindrical_grasp()
rospy.loginfo("Fall detected -- Object regrasped")
detected = True
rate.sleep()
if rospy.is_shutdown() :
break
Yesterday I came out with this code:
#!/usr/bin/env python
import struct
import rospy
from matteo.msg import acc
import struct
import os
import time
i = 0
# Mouse read with a non blocking structure, the problem is that does not provide the same output as
# mouse_clean.py, probably there is a problem with the unpacking or the reading.
while i < 1000:
i += 1
try:
file = os.open("/dev/input/mouse0", os.O_RDONLY | os.O_NONBLOCK)
time.sleep(0.1)
buf = os.read(file , 3)
x,y = struct.unpack( "bb", buf[1:] ) # <--- X and Y deltas.
print ( "X:" +str ( x ) + "---" +"Y:" +str ( y ) )
except OSError as err:
if err.errno == 11:
print ( "No motion detected")
continue
os.close(file)
It works fine, if there is no motion the message is printed out but, in case of motion the output I get is quite different from the "vanilla" mode.
I've been trying to run a code using MPI I/O on a large number of cores. The time required for each core to read from and write to a single file (the same for all cores) increases with the number of cores used. I'm currently using 512 cores and this problem is making my project unfeasible. The problem appears, however, even when running on 8 cores; it then takes about 0.2 seconds to read the first real number in the file. On 32 cores it takes more then 30 seconds to write one real number. I'm running it here: https://www.msi.umn.edu/hpc/itasca. The following simple code generates exactly this problem (the counting of the number of elements in the file might seem unnecessary here but it is necessary in my actual code):
PROGRAM MAIN
USE MPI
IMPLICIT NONE
! INITIALIZING VARIABLES
REAL(8) :: A, B
INTEGER :: COUNT_IO, i, j, ST, GO, tag, t, nb_bytes, N, d_each, d_start, d_end, NN
REAL(8) :: time_start, time_end
! VARIABLES RELATED TO MPI
INTEGER :: ierror ! returns error messages from the mpi subroutines
INTEGER :: rank ! identification number of each processor
INTEGER :: nproc ! number of processors
INTEGER, DIMENSION(mpi_status_size):: status
INTEGER(kind= MPI_OFFSET_KIND ) :: offset
INTEGER :: fh ! file handle
! EXECUTABLE
! INITIALIZE THE MPI ENVIRONMENT
CALL MPI_INIT(ierror) ! initialize MPI
CALL MPI_COMM_RANK(MPI_COMM_WORLD,rank,ierror) ! obtain rank for each node
CALL MPI_COMM_SIZE(MPI_COMM_WORLD,nproc,ierror) ! obtain the number of nodes
CALL MPI_TYPE_SIZE(MPI_REAL8,nb_bytes,ierror)
CALL MPI_FILE_OPEN (MPI_COMM_WORLD,"file.dat",MPI_MODE_RDWR+MPI_MODE_UNIQUE_OPEN,MPI_INFO_NULL,fh,ierror)
NN = 2048
DO d_each=1,NN
IF (d_each*nproc>=NN) EXIT
END DO
d_start = rank*d_each+1
d_end = MIN((rank+1)*d_each,NN)
DO t = d_start,d_end
! READING ONE THREAD AT A TIME
tag = 1
GO = 0
IF (rank .gt. 0) THEN
CALL MPI_RECV (GO,1,MPI_INTEGER,rank-1,tag, MPI_COMM_WORLD ,status,ierror)
ENDIF
time_start = MPI_WTIME()
i = 0
ST = 0
COUNT_IO = 0
DO WHILE ((i .lt. 100000) .AND. (ST .eq. 0))
i = i+1
offset = nb_bytes*(i-1)
CALL MPI_FILE_READ_AT (fh,offset,A,1,MPI_REAL8,status,ierror)
IF (status(1) .eq. 0) THEN
COUNT_IO = i
ST = 1
ELSE
COUNT_IO = 0
END IF
ENDDO
N = (COUNT_IO - 1)
IF (N .gt. 0) THEN
offset = 0
CALL MPI_FILE_READ_AT (fh,offset,B,1,MPI_REAL8,status,ierror)
ENDIF
time_end = MPI_WTIME()
PRINT *, 'My rank is', rank, 'Time for read =',time_end-time_start
GO = 1
IF (rank .lt. nproc-1) THEN
CALL MPI_SEND (GO,1, MPI_INTEGER ,rank+1,tag, MPI_COMM_WORLD ,ierror)
ENDIF
CALL MPI_BARRIER(MPI_COMM_WORLD,ierror)
! WRITING ONE THREAD AT A TIME
tag = 2
GO = 0
IF (rank .gt. 0) THEN
CALL MPI_RECV (GO,1,MPI_INTEGER,rank-1,tag, MPI_COMM_WORLD ,status,ierror)
ENDIF
time_start = MPI_WTIME()
i = 0
ST = 0
COUNT_IO = 0
DO WHILE ((i .lt. 100000) .AND. (ST .eq. 0))
i = i+1
offset = nb_bytes*(i-1)
CALL MPI_FILE_READ_AT (fh,offset,A,1,MPI_REAL8,status,ierror)
IF (status(1) .eq. 0) THEN
COUNT_IO = i
ST = 1
ELSE
COUNT_IO = 0
END IF
ENDDO
N = (COUNT_IO - 1)
offset = nb_bytes*N
CALL MPI_FILE_WRITE_AT (fh,offset,0.0D0,1,MPI_REAL8,status,ierror)
time_end = MPI_WTIME()
PRINT *, 'My rank is', rank, 'Time for write =',time_end-time_start
GO = 1
IF (rank .lt. nproc-1) THEN
CALL MPI_SEND (GO,1, MPI_INTEGER ,rank+1,tag, MPI_COMM_WORLD ,ierror)
ENDIF
CALL MPI_BARRIER(MPI_COMM_WORLD,ierror)
ENDDO
CALL MPI_FILE_CLOSE (fh,ierror)
CALL MPI_FINALIZE(ierror)
END PROGRAM MAIN
The main thing to realize here is that you can read in the data in one fell swoop (or, if memory is a problem, in chunks - but it can be in much larger chunks than individual doubles!) and that you don't need to skip to the end of the file one double at a time.
Here's an example which will read in the data in arbitrary chunk sizes, processes the data as you will, and appends some data (in this case, everyone just adds 4 copies of their rank to the end of the file). For simplicity, little python scripts help with writing and displaying test data.
$ ./writedata.py
$ ./readdata.py
[ 0. 1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12. 13. 14.
15. 16. 17. 18. 19. 20. 21. 22. 23. 24.]
$ mpirun -np 3 ./usepario
rank: 0 got data: 0.000... 24.000
rank: 1 got data: 0.000... 24.000
rank: 2 got data: 0.000... 24.000
$ ./readdata.py
[ 0. 1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12. 13. 14.
15. 16. 17. 18. 19. 20. 21. 22. 23. 24. 0. 0. 0. 0. 1.
1. 1. 1. 2. 2. 2. 2.]
usepario.f90:
module pario
contains
function openFile(filename)
use mpi
implicit none
integer :: openFile, ierr
character(len=*) :: filename
integer(MPI_OFFSET_KIND) :: off = 0
call MPI_File_open(MPI_COMM_WORLD, filename, &
ior(MPI_MODE_RDWR, MPI_MODE_UNIQUE_OPEN), &
MPI_INFO_NULL, openFile, ierr)
call MPI_File_set_view(openFile, off, &
MPI_DOUBLE_PRECISION, MPI_DOUBLE_PRECISION, &
"native", MPI_INFO_NULL, ierr)
end function openFile
subroutine closeFile(fh)
use mpi
implicit none
integer :: fh, ierr
call MPI_File_close(fh, ierr)
end subroutine closeFile
function filesizedoubles(fh)
use mpi
implicit none
integer :: fh, ierr
integer(MPI_OFFSET_KIND) :: filesize, filesizedoubles
integer :: dblsize
call MPI_File_get_size(fh, filesize, ierr)
call MPI_type_size(MPI_DOUBLE_PRECISION, dblsize, ierr)
filesizedoubles = filesize / dblsize
end function filesizedoubles
subroutine getdatablock(fh, blocksize, datablock, datasize)
use mpi
implicit none
integer :: fh, ierr
integer :: blocksize, datasize
double precision, dimension(:) :: datablock
integer(MPI_OFFSET_KIND) :: fileloc
integer, dimension(MPI_STATUS_SIZE) :: rstatus
! you can also experiment with read_all for non collective/synchronous file
! access
call MPI_File_read(fh, datablock, blocksize, MPI_DOUBLE_PRECISION, &
rstatus, ierr)
call MPI_Get_count(rstatus, MPI_DOUBLE_PRECISION, datasize, ierr)
end subroutine getdatablock
subroutine eachappend(fh, filesize, numitems, newdata)
use mpi
implicit none
integer :: fh, numitems
integer(MPI_OFFSET_KIND) :: filesize
double precision, dimension(:) :: newdata
integer :: rank, ierr
integer(MPI_OFFSET_KIND) :: offset
call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr)
offset = filesize + rank*numitems
call MPI_File_write_at_all(fh, offset, newdata, numitems, &
MPI_DOUBLE_PRECISION, &
MPI_STATUS_IGNORE, ierr)
end subroutine eachappend
end module pario
program usepario
use mpi
use pario
implicit none
integer :: fileh
integer, parameter :: bufsize=1000, newsize=4
integer(MPI_OFFSET_KIND) :: filesize
double precision, allocatable, dimension(:) :: curdata, newdata
integer :: datasize
integer :: rank, ierr
call MPI_Init(ierr)
call MPI_Comm_rank(MPI_COMM_WORLD, rank, ierr)
allocate(curdata(bufsize))
fileh = openFile("data.dat")
filesize = filesizedoubles(fileh)
do
call getdatablock(fileh, bufsize, curdata, datasize)
!!
!! process data here
!!
!! do i=1,datasize
!! ...dostuff...
!! end do
!!
print '(1X,A,I3,A,F8.3,A,F8.3)', 'rank: ', rank, ' got data: ', curdata(1), '...', curdata(datasize)
if (datasize /= bufsize) exit
end do
deallocate(curdata)
allocate(newdata(newsize))
newdata = rank
call eachappend(fileh, filesize, newsize, newdata)
call closeFile(fileh)
call MPI_Finalize(ierr)
end program usepario
writedata.py:
#!/usr/bin/env python
import numpy
numdoubles = 25
data = numpy.arange(numdoubles,dtype=numpy.float64)
data.tofile("data.dat")
readdata.py:
#!/usr/bin/env python
import numpy
data = numpy.fromfile("data.dat",dtype=numpy.float64)
print data