What I'm trying to do is take numerous(up to 48) 1024x768 images that are color coded images(weather maps, the precip overlay) and add up the precip to fall over the course of time. When I run into non-precip I want to take a box 5x5 around the pixel in question and average the value and use that value as the value of the pixel in question.
I can do this but it takes a long time to accomplish it. I have heard numpy could improve the speed but I still haven't been able to wrap my mind around how it going to improve the speed given the sequence of events that have to take place. It seems like I would still have to do it pixel by pixel. I've included an idea of the code I'm using to accomplish this SLOWLY.
I have this actually as two separate program, one to download the images and the other does the image processing(working up toward merging the two programs in the near future, just trying to get all the bugs worked out before the merger.) Hence some of the download coding may look a little strange. I figure I could probably write the file straight to a variable but I haven't been doing it that way so I stuck with a bit longer approach.
Is there anyway of increasing the speed? I don't see anyway of avoiding pixel by pixel due to the color coding scheme in place(look at the color bar in the lower left it shows the full color scheme...I only included part of it for demo purposes in the coding below.) Some of the coding may be a bit rough since I chopped from the two programs and put the important parts in here...it shows what I'm currently doing and gives the full idea of how I'm going about doing it.
Also, if you happen to see this three to four or more days after it was posted you would need to change the date in the download link to the current date. The files are only kept on the server for 3-4 days before they are removed.
from PIL import Image
import time
import urllib
import os
pathstr = '/'
url = 'http://mag.ncep.noaa.gov/GemPakTier/MagGemPakImages/gfs/20140216/00/gfs_namer_006_1000_500_thick.gif'
urllib.urlretrieve(url,str(pathstr + '20140216006.gif'))
url = 'http://mag.ncep.noaa.gov/GemPakTier/MagGemPakImages/gfs/20140216/00/gfs_namer_012_1000_500_thick.gif'
urllib.urlretrieve(url,str(pathstr + '20140216012.gif'))
url = 'http://mag.ncep.noaa.gov/GemPakTier/MagGemPakImages/gfs/20140216/00/gfs_namer_018_1000_500_thick.gif'
urllib.urlretrieve(url,str(pathstr + '20140216018.gif'))
url = 'http://mag.ncep.noaa.gov/GemPakTier/MagGemPakImages/gfs/20140216/00/gfs_namer_024_1000_500_thick.gif'
urllib.urlretrieve(url,str(pathstr + '20140216024.gif'))
class Convert():
def __init__(self):
self.colorscale2 = [(255,255,255),(127,255,0),(0,205,0),(145,44,238),(16,78,139),
(30,144,255),(0,178,238),(0,238,238),(137,104,205),(0,139,0),
(139,0,139),(139,0,0),(205,0,0),(238,64,0),(255,127,0),(205,133,0),
(255,215,0),(238,238,0),(255,255,0),(139,71,38),(255,0,0),(0,0,255),(0,0,0)]
self.x = 0
self.y = 0
self.grid = 0
self.moist = 0
self.scan = 0
self.turn = 0
self.precip = {}
start = time.time()
for i in range(6, 30, 6):
if i < 10:
filename = '/2014021600' + str(i) + '.gif'
else:
filename = '/201402160' + str(i) + '.gif'
self.im1 = Image.open(filename).convert('RGB')
self.image = self.im1.getdata()
self.size = width, height = self.im1.size
self.coordinates = self.x,self.y = width, height
self.getprecip()
self.turn = 1
print (time.time()-start)
def getprecip(self):
for self.x in range(81, 950):
for self.y in range(29, 749):
if self.turn == 0:
self.moist = 0
else:
self.moist = self.precip[self.x,self.y]
self.coordinates = self.x,self.y
self.scan = 0
self.imagescan()
if self.turn == 0:
self.precip[self.x,self.y] = self.moist
else:
self.precip[self.x,self.y] += self.moist
def imagescan(self):
if self.image[(self.y * 1024) + self.x] == self.colorscale2[0]:
self.moist =0
self.grid -=1
elif self.image[(self.y * 1024) + self.x] == self.colorscale2[1]:
self.moist =.01
elif self.image[(self.y * 1024) + self.x] == self.colorscale2[2]:
self.moist =.1
elif self.image[(self.y * 1024) + self.x] == self.colorscale2[3]:
self.moist =.25
elif self.image[(self.y * 1024) + self.x] == self.colorscale2[4]:
self.moist =.5
#on and on through self.colorscale2[18]
if self.scan == 1:
self.grid += 1
if self.scan == 0:
x = self.x
y = self.y
self.deliso540()
self.x = x
self.y = y
def deliso540(self):
self.grid = 1
self.scan = 1
for p in range(self.x-2,self.x+2):
for q in range(self.y-2,self.y+2):
self.x = p
self.y = q
self.imagescan()
self.moist = self.moist / self.grid
Related
I have this plot:
WMA_point = wma(close, 9)
plot(WMA_point, title='WMA', color=#000000)
It makes a line on the chart. When the price goes up, the line moves up, and vice versa, when the price goes down, it moves down.
I know that it depends on the "close" data.
I would like to find the price data of this wma when (close == open)
EDIT:
I mean: I want to get the first data of the close line wma(close, 9), when the new candle appears, when the (close == open) . I dont need the data of the open line wma(open, 9)
You can either use WMA_point[1], which is generally where the price value of the current bar will open at. If you want to take into consideration gapping, you'll need to calculate the wma using n-1 historical close values and with the nth value the current bar's open.
//#version=5
indicator("wma open", overlay = true)
len = input.int(10)
float wtd_close_sum = 0.0
int denom = 0
for i = 1 to len - 1
wt = len - i
wtd_close_sum += close[i] * wt
denom += wt
wtd_close_sum += open * len
denom += len
wma_open = wtd_close_sum / denom
plot(wma_open)
as a function :
f_wma_open(_open, _close, _len) =>
float _wtd_close_sum = 0.0
int _denom = 0
for i = 1 to _len - 1
_wt = _len - i
_wtd_close_sum += _close[i] * _wt
_denom += _wt
_wtd_close_sum += _open * _len
_denom += _len
_wma_open = _wtd_close_sum / _denom
_wma_open
Try using open as your source:
WMA_point = wma(open, 9)
plot(WMA_point, title='WMA', color=#000000)
Cheers, and best of luck with your trading and coding
So, you need to store the price in a var when your condition is true.
var float p = na
WMA_point = wma(close, 9)
if (open == close)
p := WMA_point // This will have the wma value of the last time open == close
plot(WMA_point, title='WMA', color=#000000)
Below is my implementation of a2c using PyTorch. Upon learning about backpropagation in PyTorch, I have known to zero_grad() the optimizer after each update iteration. However, there is still a RunTime error on second-time backpropagation.
def torchworker(number, model):
worker_env = gym.make("Taxi-v3").env
max_steps_per_episode = 2000
worker_opt = optim.Adam(lr=5e-4, params=model.parameters())
p_history = []
val_history = []
r_history = []
running_reward = 0
episode_count = 0
under = 0
start = time.time()
for i in range(2):
state = worker_env.reset()
episode_reward = 0
penalties = 0
drop = 0
print("Episode {} begins ({})".format(episode_count, number))
worker_env.render()
criterion = nn.SmoothL1Loss()
time_solve = 0
for _ in range(1, max_steps_per_episode):
#worker_env.render()
state = torch.tensor(state, dtype=torch.long)
action_probs = model.forward(state)[0]
critic_value = model.forward(state)[1]
val_history.append((state, critic_value[0]))
# Choose action
action = np.random.choice(6, p=action_probs.detach().numpy())
p_history.append(torch.log(action_probs[action]))
# Apply chosen action
state, reward, done, _ = worker_env.step(action)
r_history.append(reward)
episode_reward += reward
time_solve += 1
if reward == -10:
penalties += 1
elif reward == 20:
drop += 1
if done:
break
# Update running reward to check condition for solving
running_reward = (running_reward * (episode_count) + episode_reward) / (episode_count + 1)
# Calculate discounted returns
returns = deque(maxlen=3500)
discounted_sum = 0
for r in r_history[::-1]:
discounted_sum = r + gamma * discounted_sum
returns.appendleft(discounted_sum)
# Calculate actor losses and critic losses
loss_actor_value = 0
loss_critic_value = 0
history = zip(p_history, val_history, returns)
for log_prob, value, ret in history:
diff = ret - value[1]
loss_actor_value += -log_prob * diff
ret_tensor = torch.tensor(ret, dtype=torch.float32)
loss_critic_value += criterion(value[1], ret_tensor)
loss = loss_actor_value + 0.1 * loss_critic_value
print(loss)
# Update params
loss.backward()
worker_opt.step()
worker_opt.zero_grad()
# Log details
end = time.time()
episode_count += 1
if episode_count % 1 == 0:
worker_env.render()
if running_reward > -50: # Condition to consider the task solved
under += 1
if under > 5:
print("Solved at episode {} !".format(episode_count))
break
I believe there may be something to do with the architecture of my AC model, so I also include it here for reference.
class ActorCriticNetwork(nn.Module):
def __init__(self, num_inputs, num_hidden, num_actions):
super(ActorCriticNetwork, self).__init__()
self.embed = nn.Embedding(500, 10)
self.fc1 = nn.Linear(10, num_hidden * 2)
self.fc2 = nn.Linear(num_hidden * 2, num_hidden)
self.c = nn.Linear(num_hidden, 1)
self.fc3 = nn.Linear(num_hidden, num_hidden)
self.a = nn.Linear(num_hidden, num_actions)
def forward(self, x):
out = F.relu(self.embed(x))
out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
critic = self.c(out)
out = F.relu(self.fc3(out.detach()))
actor = F.softmax(self.a(out), dim=-1)
return actor, critic
Would you please tell me what the mistake here is? Thank you in advance.
SOLVED: I forgot to clear the history of probabilities, action-values and rewards after iterations. It is clear why that would cause the issue, as the older elements would cause propagating through old dcgs.
I'm trying to get the average of an array of Ints using the following code:
let numbers = [1,2,3,4,5]
let avg = numbers.reduce(0) { return $0 + $1 / numbers.count }
print(avg) // 1
Which is obviously incorrect. However, if I remove the division to the outside of the closure:
let numbers = [1,2,3,4,5]
let avg = numbers.reduce(0) { return $0 + $1 } / numbers.count
print(avg) // 3
Bingo! I think I remember reading somewhere (can't recall if it was in relation to Swift, JavaScript or programming math in general) that this has something to do with the fact that dividing the sum by the length yields a float / double e.g. (1 + 2) / 5 = 0.6 which will be rounded down within the sum to 0. However I would expect ((1 + 2) + 3) / 5 = 1.2 to return 1, however it too seems to return 0.
With doubles, the calculation works as expected whichever way it's calculated, as long as I box the count integer to a double:
let numbers = [1.0,2.0,3.0,4.0,5.0]
let avg = numbers.reduce(0) { return $0 + $1 / Double(numbers.count) }
print(avg) // 3
I think I understand the why (maybe not?). But I can't come up with a solid example to prove it.
Any help and / or explanation is very much appreciated. Thanks.
The division does not yield a double; you're doing integer division.
You're not getting ((1 + 2) + 3 etc.) / 5.
In the first case, you're getting (((((0 + (1/5 = 0)) + (2/5 = 0)) + (3/5 = 0)) + (4/5 = 0)) + (5/5 = 1)) = 0 + 0 + 0 + 0 + 0 + 1 = 1.
In the second case, you're getting ((((((0 + 1) + 2) + 3) + 4) + 5) / 5) = 15 / 5 = 3.
In the third case, double precision loss is much smaller than the integer, and you get something like (((((0 + (1/5.0 = 0.2)) + (2/5.0 = 0.4)) + (3/5.0 = 0.6)) + (4/5.0 = 0.8)) + (5/5.0 = 1.0)).
The problem is that what you are attempting with the first piece of code does not make sense mathematically.
The average of a sequence is the sum of the entire sequence divided by the number of elements.
reduce calls the lambda function for every member of the collection it is being called on. Thus you are summing and dividing all the way through.
For people finding it hard to understand the original answer.
Consider.
let x = 4
let y = 3
let answer = x/y
You expect the answer to be a Double, but no, it is an Int. For you to get an answer which is not a rounded down Int. You must explicitly state the values to be Double. See below
let doubleAnswer = Double(x)/Double(y)
Hope this helped.
I'm porting and updating an old app and I came across this function. I'd like to know more about it, but I don't actually know what it's called. I'm assuming it has a popular name. Does anyone know?
This version is in Python, although it was originally in Java.
def encode(msg): # msg is a string
msg_len = len(msg)
j = (msg_len + 6) / 7
k = 0
cbytesOutput = [ctypes.c_byte(0)]*(msg_len + j) # return is msg length + j bytes long
for l in xrange(j):
i1 = l * 8
j1 = i1
byte0 = ctypes.c_byte(-128)
byte1 = ctypes.c_byte(1)
k1 = 0
while k1 < 7 and k < msg_len:
byte2 = ctypes.c_byte(ord(msg[k]))
if (byte2.value & 0xffffff80) != 0:
byte0 = ctypes.c_byte(byte0.value | byte1.value)
j1 += 1
cbytesOutput[j1] = ctypes.c_byte(byte2.value | 0xffffff80)
byte1 = ctypes.c_byte(byte1.value << 1)
k += 1
k1 += 1
cbytesOutput[i1] = byte0
return cbytesOutput
Any comments on the algorithm in general? I'm thinking of replacing it. Profiler says it's the worst function in the entire app (it's 60% of the time on the slowest code path) and it bloats the data as well.
Thanks
I am involved in a project that I think you can help me. I have multiple images that you can see here Images to recognize. The goal here is to extract the numbers between the dashed lines. What is the best approach to do that? The idea that I have from the beginning is to find the coordinates of the dash lines and do the crop function, then is just run OCR software. But is not easy to find those coordinates, can you help me? Or if you have a better approach tell me.
Best regards,
Pedro Pimenta
You may start by looking at more obvious (bigger) objects in your images. The dashed lines are way too small in some images. Searching for the "euros milhoes" logo and the barcode will be easier and it will help you have an idea of the scale and rotation involved.
To find these objects without using match template you can binarize your image (watch out for the background texture) and use the Hu moments on the contours/blobs.
Don't expect a good OCR accuracy on images where the numbers are smaller than 8-10 pixels.
You can use python-tesseract https://code.google.com/p/python-tesseract/ ,it works with your image.What you need to do is to split the result string.I use your https://www.dropbox.com/sh/kcybs1i04w3ao97/u33YGH_Kv6#f:euro9.jpg to test.And source code is below.UPDATE
# -*- coding: utf-8 -*-
from PIL import Image
from PIL import ImageEnhance
import tesseract
im = Image.open('test.jpg')
enhancer = ImageEnhance.Contrast(im)
im = enhancer.enhance(4)
im = im.convert('1')
w, h = im.size
im = im.resize((w * (416 / h), 416))
pix = im.load()
LINE_CR = 0.01
WHITE_HEIGHT_CR = int(h * (20 / 416.0))
status = 0
white_line = []
for i in xrange(h):
line = []
for j in xrange(w):
line.append(pix[(j, i)])
p = line.count(0) / float(w)
if not p > LINE_CR:
white_line.append(i)
wp = None
for i in range(10, len(white_line) - WHITE_HEIGHT_CR):
k = white_line[i]
if white_line[i + WHITE_HEIGHT_CR] == k + WHITE_HEIGHT_CR:
wp = k
break
result = []
flag = 0
while 1:
if wp < 0:
result.append(wp)
break
line = []
for i in xrange(w):
line.append(pix[(i, wp)])
p = line.count(0) / float(w)
if flag == 0 and p > LINE_CR:
l = []
for xx in xrange(20):
l.append(pix[(xx, wp)])
if l.count(0) > 5:
break
l = []
for xx in xrange(416-1, 416-100-1, -1):
l.append(pix[(xx, wp)])
if l.count(0) > 17:
break
result.append(wp)
wp -= 1
flag = 1
continue
if flag == 1 and p < LINE_CR:
result.append(wp)
wp -= 1
flag = 0
continue
wp -= 1
result.reverse()
for i in range(1, len(result)):
if result[i] - result[i - 1] < 15:
result[i - 1] = -1
result = filter(lambda x: x >= 0, result)
im = im.crop((0, result[0], w, result[-1]))
im.save('test_converted.jpg')
api = tesseract.TessBaseAPI()
api.Init(".","eng",tesseract.OEM_DEFAULT)
api.SetVariable("tessedit_char_whitelist", "0123456789abcdefghijklmnopqrstuvwxyz")
api.SetPageSegMode(tesseract.PSM_AUTO)
mImgFile = "test_converted.jpg"
mBuffer=open(mImgFile,"rb").read()
result = tesseract.ProcessPagesBuffer(mBuffer,len(mBuffer),api)
print "result(ProcessPagesBuffer)=",result
Depends python 2.7 python-tesseract-win32 python-opencv numpy PIL,and be sure to follow python-tesseract's remember to .