Consider a problem:
split file by lines
write lines to a result file
if a result file exceeds some size create a new result file
For example, if I have a file which weights 4gb and split size is equal 1gb. The result is four files weights 1gb.
I'm looking for a solution with something like Rx*/Bacon or any other similar library in any language.
My solution in Coffee with Highland.js:
_ = require('underscore')
H = require('highland')
fs = require('fs')
debug = require('debug')
log = debug('main')
assert = require('assert')
readS = H(fs.createReadStream('walmart.dump')).map((buffer) ->
{ buffer: buffer }
)
MAX_SIZE = 10 ** 7
counter = 0
nextStream = ()->
stream = fs.createWriteStream("result/data#{counter}.txt")
wrapper = H.wrapCallback(stream.write.bind(stream))
counter += 1
return wrapper
debug('profile')('start')
s = readS.scan({
size: 0
stream: nextStream()
}, (acc, {buffer}) ->
debug('scan')(acc, buffer)
acc.size += buffer.length
acc.buffer = buffer
if acc.size > MAX_SIZE
debug('notify')(counter - 1, acc.size)
acc.size = 0
acc.stream = nextStream()
log(acc)
return acc
).filter((x)->x.buffer?)
s.parallel 4
s.flatMap((x) ->
debug('flatMap')(x)
x.stream(x.buffer)
)
.done -> debug('profile')('finish')
walmart.dump is a text file which contains 6gb of text. Splitting for 649 files takes:
profile start +0ms
profile finish +53s
Related
Below is my implementation of a2c using PyTorch. Upon learning about backpropagation in PyTorch, I have known to zero_grad() the optimizer after each update iteration. However, there is still a RunTime error on second-time backpropagation.
def torchworker(number, model):
worker_env = gym.make("Taxi-v3").env
max_steps_per_episode = 2000
worker_opt = optim.Adam(lr=5e-4, params=model.parameters())
p_history = []
val_history = []
r_history = []
running_reward = 0
episode_count = 0
under = 0
start = time.time()
for i in range(2):
state = worker_env.reset()
episode_reward = 0
penalties = 0
drop = 0
print("Episode {} begins ({})".format(episode_count, number))
worker_env.render()
criterion = nn.SmoothL1Loss()
time_solve = 0
for _ in range(1, max_steps_per_episode):
#worker_env.render()
state = torch.tensor(state, dtype=torch.long)
action_probs = model.forward(state)[0]
critic_value = model.forward(state)[1]
val_history.append((state, critic_value[0]))
# Choose action
action = np.random.choice(6, p=action_probs.detach().numpy())
p_history.append(torch.log(action_probs[action]))
# Apply chosen action
state, reward, done, _ = worker_env.step(action)
r_history.append(reward)
episode_reward += reward
time_solve += 1
if reward == -10:
penalties += 1
elif reward == 20:
drop += 1
if done:
break
# Update running reward to check condition for solving
running_reward = (running_reward * (episode_count) + episode_reward) / (episode_count + 1)
# Calculate discounted returns
returns = deque(maxlen=3500)
discounted_sum = 0
for r in r_history[::-1]:
discounted_sum = r + gamma * discounted_sum
returns.appendleft(discounted_sum)
# Calculate actor losses and critic losses
loss_actor_value = 0
loss_critic_value = 0
history = zip(p_history, val_history, returns)
for log_prob, value, ret in history:
diff = ret - value[1]
loss_actor_value += -log_prob * diff
ret_tensor = torch.tensor(ret, dtype=torch.float32)
loss_critic_value += criterion(value[1], ret_tensor)
loss = loss_actor_value + 0.1 * loss_critic_value
print(loss)
# Update params
loss.backward()
worker_opt.step()
worker_opt.zero_grad()
# Log details
end = time.time()
episode_count += 1
if episode_count % 1 == 0:
worker_env.render()
if running_reward > -50: # Condition to consider the task solved
under += 1
if under > 5:
print("Solved at episode {} !".format(episode_count))
break
I believe there may be something to do with the architecture of my AC model, so I also include it here for reference.
class ActorCriticNetwork(nn.Module):
def __init__(self, num_inputs, num_hidden, num_actions):
super(ActorCriticNetwork, self).__init__()
self.embed = nn.Embedding(500, 10)
self.fc1 = nn.Linear(10, num_hidden * 2)
self.fc2 = nn.Linear(num_hidden * 2, num_hidden)
self.c = nn.Linear(num_hidden, 1)
self.fc3 = nn.Linear(num_hidden, num_hidden)
self.a = nn.Linear(num_hidden, num_actions)
def forward(self, x):
out = F.relu(self.embed(x))
out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
critic = self.c(out)
out = F.relu(self.fc3(out.detach()))
actor = F.softmax(self.a(out), dim=-1)
return actor, critic
Would you please tell me what the mistake here is? Thank you in advance.
SOLVED: I forgot to clear the history of probabilities, action-values and rewards after iterations. It is clear why that would cause the issue, as the older elements would cause propagating through old dcgs.
I am working on code where I need to hash values. SipHash seems like a great option.
let getSipHashValue (buffer:byte []) (key:byte []) =
match key.GetLength(0) with
| 16 -> SipHash24.Hash64(buffer, key)
| _ -> uint64(0)
Is there a way to pad the key to 16 bytes and make sure that it works?
I can get the exact length word as key but I would like to be able to use any word (that is shorter than 16 bytes) and just use some padding.
open System
open System.Text
let testKey : byte [] =
Encoding.UTF8.GetBytes "accumulativeness"
Console.WriteLine("Length: {0}", testKey.GetLength(0))
Is there a way to do that in F#?
I think I got it:
open System
open System.Text
let rec getPaddedBytes (s:string) =
let b = Encoding.UTF8.GetBytes s
match b.GetLength(0) with
| 16 -> b
| x when x < 16 -> getPaddedBytes (s + "0")
| _ -> b[0..15]
Console.WriteLine("Length: {0}", testKey.GetLength(0))
let testBytes = getPaddedBytes "accum"
let testString = Encoding.UTF8.GetString testBytes
Console.WriteLine("X: {0}", testString)
I need to fix getting the first 16 bytes. Not sure about that syntax.
It's not clear from your question what you want the padding to look like but you can pad with zeros with:
let getPadding (bs: byte[]): byte[] =
let rem = bs.Length % 16
let padBytes = if rem = 0 then 0 else (16 - rem)
Array.zeroCreate padBytes
let pad (bs: byte[]): byte[] =
Array.append bs (getPadding bs)
which you can then use with:
let padded = pad testKey
printfn "Key length: %d" padded.Length
I am currently trying to take a 15MB video file that is stored locally on my machine and write that file into chunks of 5MB in binary file format. I am trying to get this to work for any file size to where it splits up larger file formats into smaller 5MB Chunks.
val chunksize: Int = 5242880
val byteArray = Files.readAllBytes(Paths.get("/path/to/inputfile"))
val fos = new FileOutputStream("/path/to/outputfile")
val ret = new Array[Array[Byte]](Math.ceil(byteArray.length / chunksize.asInstanceOf[Double]).toInt, chunksize)
var start = 0
var i = 0
while ( {
i < ret.length
}) {
ret(i) = util.Arrays.copyOfRange(byteArray, start, start + chunksize)
start += chunksize
//Here is where i need to write the bytes to a file output stream so I can get a binary file of less than 5MB
{
i += 1; i - 1
}
}
fos.close()
Executing "3Dball" creates some errors in Unity ml-agent
When I execute PPO.ipynb, there is no error till "Load the environment".
Executing "Train the Agents" there are some errors
ValueError: List argument 'values' to 'ConcatV2' Op with length 0
shorter than minimum length 2.
This is the code I executed
https://github.com/Unity-Technologies/ml-agents/blob/master/python/PPO.ipynb
tf.reset_default_graph()
if curriculum_file == "None":
curriculum_file = None
def get_progress():
if curriculum_file is not None:
if env._curriculum.measure_type == "progress":
return steps / max_steps
elif env._curriculum.measure_type == "reward":
return last_reward
else:
return None
else:
return None
# Create the Tensorflow model graph
ppo_model = create_agent_model(env, lr=learning_rate,
h_size=hidden_units, epsilon=epsilon,
beta=beta, max_step=max_steps,
normalize=normalize, num_layers=num_layers)
is_continuous = (env.brains[brain_name].action_space_type == "continuous")
use_observations = (env.brains[brain_name].number_observations > 0)
use_states = (env.brains[brain_name].state_space_size > 0)
model_path = './models/{}'.format(run_path)
summary_path = './summaries/{}'.format(run_path)
if not os.path.exists(model_path):
os.makedirs(model_path)
if not os.path.exists(summary_path):
os.makedirs(summary_path)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
# Instantiate model parameters
if load_model:
print('Loading Model...')
ckpt = tf.train.get_checkpoint_state(model_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
sess.run(init)
steps, last_reward = sess.run([ppo_model.global_step, ppo_model.last_reward])
summary_writer = tf.summary.FileWriter(summary_path)
info = env.reset(train_mode=train_model, progress=get_progress())[brain_name]
trainer = Trainer(ppo_model, sess, info, is_continuous, use_observations, use_states, train_model)
if train_model:
trainer.write_text(summary_writer, 'Hyperparameters', hyperparameter_dict, steps)
while steps <= max_steps:
if env.global_done:
info = env.reset(train_mode=train_model, progress=get_progress())[brain_name]
# Decide and take an action
new_info = trainer.take_action(info, env, brain_name, steps, normalize)
info = new_info
trainer.process_experiences(info, time_horizon, gamma, lambd)
if len(trainer.training_buffer['actions']) > buffer_size and train_model:
# Perform gradient descent with experience buffer
trainer.update_model(batch_size, num_epoch)
if steps % summary_freq == 0 and steps != 0 and train_model:
# Write training statistics to tensorboard.
trainer.write_summary(summary_writer, steps, env._curriculum.lesson_number)
if steps % save_freq == 0 and steps != 0 and train_model:
# Save Tensorflow model
save_model(sess, model_path=model_path, steps=steps, saver=saver)
steps += 1
sess.run(ppo_model.increment_step)
if len(trainer.stats['cumulative_reward']) > 0:
mean_reward = np.mean(trainer.stats['cumulative_reward'])
sess.run(ppo_model.update_reward, feed_dict={ppo_model.new_reward: mean_reward})
last_reward = sess.run(ppo_model.last_reward)
# Final save Tensorflow model
if steps != 0 and train_model:
save_model(sess, model_path=model_path, steps=steps, saver=saver)
env.close()
export_graph(model_path, env_name)
I had the same error, the way I fixed it is by replacing line 222 under the file: "ml-agents/python/ppo/models.py":
REPLACE Line 222:
hidden_visual = tf.concat(encoders, axis=2)
BY:
if encoders:
hidden_visual = tf.concat(encoders, axis=2)
I hope that helped you.
I'm porting and updating an old app and I came across this function. I'd like to know more about it, but I don't actually know what it's called. I'm assuming it has a popular name. Does anyone know?
This version is in Python, although it was originally in Java.
def encode(msg): # msg is a string
msg_len = len(msg)
j = (msg_len + 6) / 7
k = 0
cbytesOutput = [ctypes.c_byte(0)]*(msg_len + j) # return is msg length + j bytes long
for l in xrange(j):
i1 = l * 8
j1 = i1
byte0 = ctypes.c_byte(-128)
byte1 = ctypes.c_byte(1)
k1 = 0
while k1 < 7 and k < msg_len:
byte2 = ctypes.c_byte(ord(msg[k]))
if (byte2.value & 0xffffff80) != 0:
byte0 = ctypes.c_byte(byte0.value | byte1.value)
j1 += 1
cbytesOutput[j1] = ctypes.c_byte(byte2.value | 0xffffff80)
byte1 = ctypes.c_byte(byte1.value << 1)
k += 1
k1 += 1
cbytesOutput[i1] = byte0
return cbytesOutput
Any comments on the algorithm in general? I'm thinking of replacing it. Profiler says it's the worst function in the entire app (it's 60% of the time on the slowest code path) and it bloats the data as well.
Thanks