MiniTest falis with jruby rake test task - rake

I have a project with self written gems in jruby (using rvm to switch between jrubies). With jruby-1.7.0 I could do my tests with Test::Unit::TestCase. Now I have switched to MiniTest in jruby-1.7.13. But I cannot run my tests any more with 'rake test'.
The errors are like ... Mocha::ExpectationError: unexpected invocation: blabla.new() ... with blabla beeing my jruby class
It only works if I run them solely via the TEST= and TESTOPTS= parameter to use a single test file and select one or a view tests via --name= (with regular expressions).
I use a helper file with
require 'minitest/spec'
require 'minitest/autorun'
require 'mocha/integration'
class MiniTest::Test
end
instead of the lines used for the old test suite Test::Unit::TestCase
require 'test/unit'
require 'shoulda'
require 'mocha'
class Test::Unit::TestCase
end
A major difference in MiniTest is the test order. So I tried to circumvent the randomization by definig the test_order like
class MiniTest::Test
def test_order
:alpha
end
end
this had no effect on the test order, but Is this realy a problem? The errors seem to come with too may tests in the test suite. Is this a bug? Please help!
You may also check
require self made gem in jruby fails after update to jruby-1.7.13
for my preious post where I already could solve a load problem.

Here is some code
require File.dirname(__FILE__) + "/../helper.rb"
require 'command/flow_control_command'
## --seed=36725
module Command
class TestFlowControlCommand < MiniTest::Test
def setup
fcp = Configuration::FlowControlParameter.last
fcp.number_ports = 16
fcp.save!
end
def test_should_exist
assert FlowControlCommand
end
def test_should_initialize
FlowControlCommand.expects(:new).with("modbus").returns(true)
assert FlowControlCommand.new("modbus")
end
def test_should_have_constants
assert_equal [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0], FlowControlCommand::CIRCLE
end
def test_should_find_line_1
result = FlowControlCommand.new("modbus").find_line [1,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 1,1,0,0, 0,0]
assert_equal "LINE1", result
end
def test_should_not_find_line
result = FlowControlCommand.new("modbus").find_line [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 1,1,0,0, 0,0]
assert_equal "LINE?", result
result = FlowControlCommand.new("modbus").find_line [1,1,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 1,1,0,0, 0,0]
assert_equal "LINE?", result
end
def test_should_find_line_16
result = FlowControlCommand.new("modbus").find_line [0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,1, 1,1,0,0, 0,0]
assert_equal "LINE16", result
end
def test_should_find_line_13
result = FlowControlCommand.new("modbus").find_line [0,0,0,0, 0,0,0,0, 0,0,0,0, 1,0,0,0, 1,1,0,0, 0,0]
assert_equal "LINE13", result
end
+ some more tests
Running this will cause
Running:
.....EEEEE.EEE.
Finished in 1.036000s, 14.4788 runs/s, 2.8958 assertions/s.
1) Error:
Command::TestFlowControlCommand#test_should_find_line_circle:
Mocha::ExpectationError: unexpected invocation: Command::FlowControlCommand.new('modbus')
unsatisfied expectations:
- expected exactly once, invoked twice: Command::FlowControlCommand.new('modbus')
/home/frank/Rubymineprojects/ndir_med1/dibta-model/test/command/test_flow_control_command.rb:61:in `test_should_find_line_circle'
2) Error:
Command::TestFlowControlCommand#test_should_find_line_14:
Mocha::ExpectationError: unexpected invocation: Command::FlowControlCommand.new('modbus')
unsatisfied expectations:
- expected exactly once, invoked 3 times: Command::FlowControlCommand.new('modbus')
/home/frank/Rubymineprojects/ndir_med1/dibta-model/test/command/test_flow_control_command.rb:46:in `test_should_find_line_14'
If I run the tests one by one no error occurs, If I run with a specail seed parameter the tests also run through.

Related

Using numba in a method to randomize matrices

I'm still not very familiar with numba and my problem is that I have the piece of code bellow that I use for randomize the edges of graphs.
This code is simply used to swap some edges in a connectivity matrix given the number of desired swaps and a seed for the random number generator.
My problem is that when I try to use it with numba to speed it up I did not menage to run it. The error it returns is also pasted bellow.
#nb.jit(nopython=True)
def _randomize_adjacency_wei(A, n_swaps, seed):
np.random.seed(seed)
# Number of nodes
n_nodes = A.shape[0]
# Copy the adj. matrix
Arnd = A.copy()
# Choose edges that will be swaped
edges = np.random.choice(n_nodes, size=(4, n_swaps), replace=True).T
#itr = range(n_swaps)
#for it in tqdm(itr) if verbose else itr:
it = 0
for it in range(n_swaps):
i,j,k,l = edges[it,:]
if len(np.unique([i,j,k,l]))<4:
continue
else:
# Old values of weigths
w_ij,w_il,w_kj,w_kl=Arnd[i,j],Arnd[i,l],Arnd[k,j],Arnd[k,l]
# Swaping edges
Arnd[i,j]=Arnd[j,i]=w_il
Arnd[k,l]=Arnd[l,k]=w_kj
Arnd[i,l]=Arnd[l,i]=w_ij
Arnd[k,j]=Arnd[j,k]=w_kl
return Arnd
TypingError: Failed in nopython mode pipeline (step: nopython frontend)
No implementation of function Function(<function unique at 0x7f1a1c03b0d0>) found for signature:
>>> unique(list(int64)<iv=None>)
There are 2 candidate implementations:
- Of which 2 did not match due to:
Overload in function 'np_unique': File: numba/np/arrayobj.py: Line 1915.
With argument(s): '(list(int64)<iv=None>)':
Rejected as the implementation raised a specific error:
TypingError: Failed in nopython mode pipeline (step: nopython frontend)
Unknown attribute 'ravel' of type list(int64)<iv=None>
File "../../../home/vinicius/anaconda3/lib/python3.8/site-packages/numba/np/arrayobj.py", line 1918:
def np_unique_impl(a):
b = np.sort(a.ravel())
^
During: typing of get attribute at /home/vinicius/anaconda3/lib/python3.8/site-packages/numba/np/arrayobj.py (1918)
File "../../../home/vinicius/anaconda3/lib/python3.8/site-packages/numba/np/arrayobj.py", line 1918:
def np_unique_impl(a):
b = np.sort(a.ravel())
^
raised from /home/vinicius/anaconda3/lib/python3.8/site-packages/numba/core/typeinfer.py:1071
During: resolving callee type: Function(<function unique at 0x7f1a1c03b0d0>)
During: typing of call at <ipython-input-165-90ffd30fe0e8> (19)
File "<ipython-input-165-90ffd30fe0e8>", line 19:
def _randomize_adjacency_wei(A, n_swaps, seed):
<source elided>
i,j,k,l = edges[it,:]
if len(np.unique([i,j,k,l]))<4:
^
Thanks in advance,
Vinicius
According to the comments, you are passing a list to np.unique() but this is not supported by Numba.
Modifying the code this way:
i, j, k, l = e = edges[it, :]
if len(np.unique(e)) < 4:
...
The following example doesn't produce any errors:
>>> A = np.random.randint(0, 5, (8,8))
>>> r = _randomize_adjacency_wei(A, 4, 33)

Talos --> TypeError: __init__() got an unexpected keyword argument 'grid_downsample'

I am trying to run a hyperparameters optimization with Talos. As I have a lot of parameters to test, I want to use a 'grid_downsample' argument that will select 30% of all possible hyperparameters combinations. However when I run my code I get: TypeError: __init__() got an unexpected keyword argument 'grid_downsample'
I tested the code below without the 'grid_downsample' option and with less hyperparameters.
#load data
data = pd.read_csv('data.txt', sep="\t", encoding = "latin1")
# split into input (X) and output (y) variables
Y = np.array(data['Y'])
data_bis = data.drop(['Y'], axis = 1)
X = np.array(data_bis)
p = {'activation':['relu'],
'optimizer': ['Nadam'],
'first_hidden_layer': [12],
'second_hidden_layer': [12],
'batch_size': [20],
'epochs': [10,20],
'dropout_rate':[0.0, 0.2]}
def dnn_model(x_train, y_train, x_val, y_val, params):
model = Sequential()
#input layer
model.add(Dense(params['first_hidden_layer'], input_shape=(1024,)))
model.add(Dropout(params['dropout_rate']))
model.add(Activation(params['activation']))
#hidden layer 2
model.add(Dense(params['second_hidden_layer']))
model.add(Dropout(params['dropout_rate']))
model.add(Activation(params['activation']))
# output layer with one node
model.add(Dense(1))
model.add(Activation(params['activation']))
# Compile model
model.compile(loss='binary_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])
out = model.fit(x_train, y_train,
batch_size=params['batch_size'],
epochs=params['epochs'],
validation_data=[x_val, y_val],
verbose=0)
return out, model
scan_object = ta.Scan(X, Y, model=dnn_model, params=p, experiment_name="test")
reporting = ta.Reporting(scan_object)
report = reporting.data
report.to_csv('./Random_search/dnn/report_talos.txt', sep = '\t')
This code works well. If I change the scan_object as the end to: scan_object = ta.Scan(X, Y, model=dnn_model, grid_downsample=0.3, params=p, experiment_name="test"), it gives me the error: TypeError: __init__() got an unexpected keyword argument 'grid_downsample' while I was expecting to have the same results format as a normal grid search but with less combinations. What am I missing? Did the name of the argument change? I'm using Talos 0.6.3 in a conda environment.
Thank you!
might be too late for you now but they've switched it to fraction_limit. It would give this for you
scan_object = ta.Scan(X, Y, model=dnn_model, params=p, experiment_name="test", fraction_limit = 0.1)
Sadly, the doc isn't well updated
Check out their examples on GitHub:
https://github.com/autonomio/talos/blob/master/examples/Hyperparameter%20Optimization%20with%20Keras%20for%20the%20Iris%20Prediction.ipynb

Simpy: How can I represent failures in a train subway simulation?

New python user here and first post on this great website. I haven't been able to find an answer to my question so hopefully it is unique.
Using simpy I am trying to create a train subway/metro simulation with failures and repairs periodically built into the system. These failures happen to the train but also to signals on sections of track and on plaforms. I have read and applied the official Machine Shop example (which you can see resemblance of in the attached code) and have thus managed to model random failures and repairs to the train by interrupting its 'journey time'.
However I have not figured out how to model failures of signals on the routes which the trains follow. I am currently just specifying a time for a trip from A to B, which does get interrupted but only due to train failure.
Is it possible to define each trip as its own process i.e. a separate process for sections A_to_B and B_to_C, and separate platforms as pA, pB and pC. Each one with a single resource (to allow only one train on it at a time) and to incorporate random failures and repairs for these section and platform processes? I would also need to perhaps have several sections between two platforms, any of which could experience a failure.
Any help would be greatly appreciated.
Here's my code so far:
import random
import simpy
import numpy
RANDOM_SEED = 1234
T_MEAN_A = 240.0 # mean journey time
T_MEAN_EXPO_A = 1/T_MEAN_A # for exponential distribution
T_MEAN_B = 240.0 # mean journey time
T_MEAN_EXPO_B = 1/T_MEAN_B # for exponential distribution
DWELL_TIME = 30.0 # amount of time train sits at platform for passengers
DWELL_TIME_EXPO = 1/DWELL_TIME
MTTF = 3600.0 # mean time to failure (seconds)
TTF_MEAN = 1/MTTF # for exponential distribution
REPAIR_TIME = 240.0
REPAIR_TIME_EXPO = 1/REPAIR_TIME
NUM_TRAINS = 1
SIM_TIME_DAYS = 100
SIM_TIME = 3600 * 18 * SIM_TIME_DAYS
SIM_TIME_HOURS = SIM_TIME/3600
# Defining the times for processes
def A_B(): # returns processing time for journey A to B
return random.expovariate(T_MEAN_EXPO_A) + random.expovariate(DWELL_TIME_EXPO)
def B_C(): # returns processing time for journey B to C
return random.expovariate(T_MEAN_EXPO_B) + random.expovariate(DWELL_TIME_EXPO)
def time_to_failure(): # returns time until next failure
return random.expovariate(TTF_MEAN)
# Defining the train
class Train(object):
def __init__(self, env, name, repair):
self.env = env
self.name = name
self.trips_complete = 0
self.broken = False
# Start "travelling" and "break_train" processes for the train
self.process = env.process(self.running(repair))
env.process(self.break_train())
def running(self, repair):
while True:
# start trip A_B
done_in = A_B()
while done_in:
try:
# going on the trip
start = self.env.now
yield self.env.timeout(done_in)
done_in = 0 # Set to 0 to exit while loop
except simpy.Interrupt:
self.broken = True
done_in -= self.env.now - start # How much time left?
with repair.request(priority = 1) as req:
yield req
yield self.env.timeout(random.expovariate(REPAIR_TIME_EXPO))
self.broken = False
# Trip is finished
self.trips_complete += 1
# start trip B_C
done_in = B_C()
while done_in:
try:
# going on the trip
start = self.env.now
yield self.env.timeout(done_in)
done_in = 0 # Set to 0 to exit while loop
except simpy.Interrupt:
self.broken = True
done_in -= self.env.now - start # How much time left?
with repair.request(priority = 1) as req:
yield req
yield self.env.timeout(random.expovariate(REPAIR_TIME_EXPO))
self.broken = False
# Trip is finished
self.trips_complete += 1
# Defining the failure
def break_train(self):
while True:
yield self.env.timeout(time_to_failure())
if not self.broken:
# Only break the train if it is currently working
self.process.interrupt()
# Setup and start the simulation
print('Train trip simulator')
random.seed(RANDOM_SEED) # Helps with reproduction
# Create an environment and start setup process
env = simpy.Environment()
repair = simpy.PreemptiveResource(env, capacity = 1)
trains = [Train(env, 'Train %d' % i, repair)
for i in range(NUM_TRAINS)]
# Execute
env.run(until = SIM_TIME)
# Analysis
trips = []
print('Train trips after %s hours of simulation' % SIM_TIME_HOURS)
for train in trains:
print('%s completed %d trips.' % (train.name, train.trips_complete))
trips.append(train.trips_complete)
mean_trips = numpy.mean(trips)
std_trips = numpy.std(trips)
print "mean trips: %d" % mean_trips
print "standard deviation trips: %d" % std_trips
it looks like you are using Python 2, which is a bit unfortunate, because
Python 3.3 and above give you some more flexibility with Python generators. But
your problem should be solveable in Python 2 nonetheless.
you can use sub processes within in a process:
def sub(env):
print('I am a sub process')
yield env.timeout(1)
# return 23 # Only works in py3.3 and above
env.exit(23) # Workaround for older python versions
def main(env):
print('I am the main process')
retval = yield env.process(sub(env))
print('Sub returned', retval)
As you can see, you can use Process instances returned by Environment.process()
like normal events. You can even use return values in your sub proceses.
If you use Python 3.3 or newer, you don’t have to explicitly start a new
sub-process but can use sub() as a sub routine instead and just forward the
events it yields:
def sub(env):
print('I am a sub routine')
yield env.timeout(1)
return 23
def main(env):
print('I am the main process')
retval = yield from sub(env)
print('Sub returned', retval)
You may also be able to model signals as resources that may either be used
by failure process or by a train. If the failure process requests the signal
at first, the train has to wait in front of the signal until the failure
process releases the signal resource. If the train is aleady passing the
signal (and thus has the resource), the signal cannot break. I don’t think
that’s a problem be cause the train can’t stop anyway. If it should be
a problem, just use a PreemptiveResource.
I hope this helps. Please feel welcome to join our mailing list for more
discussions.

Debugging GNU octave: Using dbnext after dbup

I'm using GNU octave 3.6.4. According to the changelog (v 3.2):
Moving up and down the call stack with
dbup and dbdown now works.
However, when I'm in debug mode and excecute dbup followed by dbnext, the next line in the lower frame will be executed. Why is this so and how can it be avoided?
octave -q
octave:1> myfunc_base(2,3)
stopped in /home/seb/octave/myfunc.m at line 5
5: keyboard
debug> dbstack
stopped in:
--> myfunc at line 5 [/home/seb/octave/myfunc.m]
myfunc_base at line 4 [/home/seb/octave/myfunc_base.m]
debug> dbup
stopped in myfunc_base at line 4 % <-- looks good!
debug> dbnext
stopped in /home/seb/octave/myfunc.m at line 6 % <-- damn this is the old frame!
6: sp = a + temp;
debug>
The two test functions:
myfunc.m
function sp = myfunc (a, b, c)
temp = b+c;
keyboard
sp = a + temp;
end
myfunc_base.m
function sp = myfunc_base (aa, bb)
temp = myfunc(aa, aa, bb);
sp = aa + temp;
end
To step out you have to use dbstep out. This matches the behaviour of matlab and everything else would be very stange. You can not step to the next line on any level of the stack if an exception occurs.

Retrying celery failed tasks that are part of a chain

I have a celery chain that runs some tasks. Each of the tasks can fail and be retried. Please see below for a quick example:
from celery import task
#task(ignore_result=True)
def add(x, y, fail=True):
try:
if fail:
raise Exception('Ugly exception.')
print '%d + %d = %d' % (x, y, x+y)
except Exception as e:
raise add.retry(args=(x, y, False), exc=e, countdown=10)
#task(ignore_result=True)
def mul(x, y):
print '%d * %d = %d' % (x, y, x*y)
and the chain:
from celery.canvas import chain
chain(add.si(1, 2), mul.si(3, 4)).apply_async()
Running the two tasks (and assuming that nothing fails), your would get/see printed:
1 + 2 = 3
3 * 4 = 12
However, when the add task fails the first time and succeeds in subsequent retry calls, the rest of the tasks in the chain do not run, i.e. the add task fails, all other tasks in the chain are not run and after a few seconds, the add task runs again and succeeds and the rest of the tasks in the chain (in this case mul.si(3, 4)) does not run.
Does celery provide a way to continue failed chains from the task that failed, onwards? If not, what would be the best approach to accomplishing this and making sure that a chain's tasks run in the order specified and only after the previous task has executed successfully even if the task is retried a few times?
Note 1: The issue can be solved by doing
add.delay(1, 2).get()
mul.delay(3, 4).get()
but I am interested in understanding why chains do not work with failed tasks.
You've found a bug :)
Fixed in https://github.com/celery/celery/commit/b2b9d922fdaed5571cf685249bdc46f28acacde3
will be part of 3.0.4.
I'm also interested in understanding why chains do not work with failed tasks.
I dig some celery code and what I've found so far is:
The implementation happends at app.builtins.py
#shared_task
def add_chain_task(app):
from celery.canvas import chord, group, maybe_subtask
_app = app
class Chain(app.Task):
app = _app
name = 'celery.chain'
accept_magic_kwargs = False
def prepare_steps(self, args, tasks):
steps = deque(tasks)
next_step = prev_task = prev_res = None
tasks, results = [], []
i = 0
while steps:
# First task get partial args from chain.
task = maybe_subtask(steps.popleft())
task = task.clone() if i else task.clone(args)
i += 1
tid = task.options.get('task_id')
if tid is None:
tid = task.options['task_id'] = uuid()
res = task.type.AsyncResult(tid)
# automatically upgrade group(..) | s to chord(group, s)
if isinstance(task, group):
try:
next_step = steps.popleft()
except IndexError:
next_step = None
if next_step is not None:
task = chord(task, body=next_step, task_id=tid)
if prev_task:
# link previous task to this task.
prev_task.link(task)
# set the results parent attribute.
res.parent = prev_res
results.append(res)
tasks.append(task)
prev_task, prev_res = task, res
return tasks, results
def apply_async(self, args=(), kwargs={}, group_id=None, chord=None,
task_id=None, **options):
if self.app.conf.CELERY_ALWAYS_EAGER:
return self.apply(args, kwargs, **options)
options.pop('publisher', None)
tasks, results = self.prepare_steps(args, kwargs['tasks'])
result = results[-1]
if group_id:
tasks[-1].set(group_id=group_id)
if chord:
tasks[-1].set(chord=chord)
if task_id:
tasks[-1].set(task_id=task_id)
result = tasks[-1].type.AsyncResult(task_id)
tasks[0].apply_async()
return result
def apply(self, args=(), kwargs={}, **options):
tasks = [maybe_subtask(task).clone() for task in kwargs['tasks']]
res = prev = None
for task in tasks:
res = task.apply((prev.get(), ) if prev else ())
res.parent, prev = prev, res
return res
return Chain
You can see that at the end prepare_steps prev_task is linked to the next task.
When the prev_task failed the next task is not called.
I'm testing with adding the link_error from prev task to the next:
if prev_task:
# link and link_error previous task to this task.
prev_task.link(task)
prev_task.link_error(task)
# set the results parent attribute.
res.parent = prev_res
But then, the next task must take care of both cases (maybe, except when it's configured to be immutable, e.g. not accept more arguments).
I think chain can support that by allowing some syntax likes this:
c = chain(t1, (t2, t1e), (t3, t2e))
which means:
t1 link to t2 and link_error to t1e
t2 link to t3 and link_error to t2e