This code works fine in an Ipython/Juypter cell to get the notebook filename:
js = """var kernel = IPython.notebook.kernel;
var thename = window.document.getElementById("notebook_name").innerHTML;
var command = "theNotebook2 = " + "'"+thename+"'";
kernel.execute(command);"""
display(Javascript(js))
theNotebook2 + '.ipynb'
'techela.ipynb'
If I define a function in a cell and call it:
def get_filename():
"""Get the notebook filename."""
js = """var kernel = IPython.notebook.kernel;
var thename = window.document.getElementById("notebook_name").innerHTML;
var command = "theNotebook2 = " + "'"+thename+"'";
kernel.execute(command);"""
display(Javascript(js))
return theNotebook2 + '.ipynb'
get_filename()
'techela.ipynb'
It also seems to work fine and give me the filename.
However, if I put that function in a module, and import it, then it stops working.
from techela import get_filename
get_filename()
---------------------------------------------------------------------------
NameError Traceback (most recent call last)
<ipython-input-17-42ee37d0d253> in <module>()
1 from techela import get_filename
----> 2 get_filename()
/Users/jkitchin/techela/ipynb/techela.py in get_filename()
11 kernel.execute(command);"""
12 display(Javascript(js))
---> 13 return theNotebook2 + '.ipynb'
14
15
NameError: name 'theNotebook2' is not defined
Any idea why this is failing?
Related
I'm using Detectron2 to do instance segmentation as in the tutorial. Below is the code:
from detectron2.config import CfgNode
import detectron2.data.transforms as T
from detectron2.data import build_detection_train_loader, DatasetMapper
os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)
transform_list = [
# T.Resize(shape=(200,300)),
T.RandomRotation(angle=90.0),
# T.RandomContrast(intensity_min=0.75, intensity_max=1.25),
# T.RandomBrightness(intensity_min=0.75, intensity_max=1.25),
# T.RandomSaturation(intensity_min=0.75, intensity_max=1.25),
# T.RandomLighting(scale=0.1),
T.RandomFlip(),
# T.RandomCrop(crop_type="absolute", crop_size=(180, 270))
]
# custom_mapper = get_custom_mapper(transfrom_list)
custom_mapper = DatasetMapper(
cfg,
is_train=True,
augmentations=transform_list,
use_instance_mask=True,
instance_mask_format="bitmask",
)
class CustomTrainer(DefaultTrainer):
#classmethod
def build_test_loader(cls, cfg: CfgNode, dataset_name):
return build_detection_test_loader(cfg, dataset_name, mapper=custom_mapper)
#classmethod
def build_train_loader(cls, cfg: CfgNode):
return build_detection_train_loader(cfg, mapper=custom_mapper)
cfg.INPUT.MASK_FORMAT = 'bitmask'
cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS = False
trainer = CustomTrainer(cfg)
# trainer = DefaultTrainer(cfg)
# trainer.resume_or_load(resume=False)
# trainer.train()
However, in this case I don't care about instances and more like I want to do semantic segmentation but there is no tutorial or examples to do that nor I'm seeing a semantic model I can start with. Misc/semantic_R_50_FPN_1x.yaml throws an error saying there is no pretrained model available.
So instead I'm trying to use the SemSegEvaluator instead of COCO evaluator to give me metrics around semantic rather than instances. Below is the code:
from detectron2.evaluation import COCOEvaluator, inference_on_dataset, SemSegEvaluator
from detectron2.data import build_detection_test_loader
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.4
# evaluator = COCOEvaluator(val_dataset_name, output_dir=os.path.join(cfg.OUTPUT_DIR, 'val'), use_fast_impl=False, tasks=['segm'])
evaluator = SemSegEvaluator(val_dataset_name, output_dir=os.path.join(cfg.OUTPUT_DIR, 'val'))
val_loader = build_detection_test_loader(cfg, val_dataset_name)
eval_result = inference_on_dataset(predictor.model, val_loader, evaluator)
print(eval_result)
However, this is failing with the following error:
[12/20 16:29:02 d2.data.datasets.coco]: Loaded 50 imagesss abdul in COCO format from /content/gdrive/MyDrive/SolarDetection/datasets/train8//val/labels.json
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-10-61bd5aaec8ea> in <module>
3 cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.4
4 # evaluator = COCOEvaluator(val_dataset_name, output_dir=os.path.join(cfg.OUTPUT_DIR, 'val'), use_fast_impl=False, tasks=['segm'])
----> 5 evaluator = SemSegEvaluator(val_dataset_name, output_dir=os.path.join(cfg.OUTPUT_DIR, 'val'))
6 val_loader = build_detection_test_loader(cfg, val_dataset_name)
7 # ipdb.set_trace(context=6)
1 frames
/content/gdrive/MyDrive/repos/detectron2/detectron2/evaluation/sem_seg_evaluation.py in <dictcomp>(.0)
69
70 self.input_file_to_gt_file = {
---> 71 dataset_record["file_name"]: dataset_record["sem_seg_file_name"]
72 for dataset_record in DatasetCatalog.get(dataset_name)
73 }
KeyError: 'sem_seg_file_name'
Any idea or hint how I can setup and use the SemSegEvaluator?
Is it possible to use pathlib.Path objects with spark.read.parquet and other pyspark.sql.DataFrameReader methods?
It doesn't work by default:
>>> from pathlib import Path
>>> basedir = Path("/data")
>>> spark.read.parquet(basedir / "name.parquet")
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-5-cec8ced1bc5d> in <module>
----> 1 spark.read.parquet(basedir / "name.parquet")
<... a long traceback ...>
/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/protocol.py in get_command_part(parameter, python_proxy_pool)
296 command_part += ";" + interface
297 else:
--> 298 command_part = REFERENCE_TYPE + parameter._get_object_id()
299
300 command_part += "\n"
AttributeError: 'PosixPath' object has no attribute '_get_object_id'
I tried to write py4j type converter:
class PathConverter(object):
def can_convert(self, object):
return isinstance(object, Path)
def convert(self, object, gateway_client):
JavaString = JavaClass("java.lang.String", gateway_client)
return JavaString(str(object))
register_input_converter(PathConverter())
But it looks like I misunderstood some string conversion related concepts/specifics, because jvm.java.lang.String("string") in py4j returns the python str object:
>>> spark.read.parquet(basedir / "name.parquet")
<... a long traceback ...>
/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py in __call__(self, *args)
1306
1307 for temp_arg in temp_args:
-> 1308 temp_arg._detach()
AttributeError: 'str' object has no attribute '_detach'
I have only one ugly solution for now:
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index fa3e829a88..7441a8ba8c 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
## -298,7 +298,7 ## class DataFrameReader(OptionUtils):
modifiedAfter=modifiedAfter, datetimeRebaseMode=datetimeRebaseMode,
int96RebaseMode=int96RebaseMode)
- return self._df(self._jreader.parquet(_to_seq(self._spark._sc, paths)))
+ return self._df(self._jreader.parquet(_to_seq(self._spark._sc, paths, converter=str)))
def text(self, paths, wholetext=False, lineSep=None, pathGlobFilter=None,
recursiveFileLookup=None, modifiedBefore=None,
Also, looking through the readwriter.py source code it feels safe enough to monkeypatch its version of _to_seq:
from pyspark.sql import readwriter
def converter(x):
if isinstance(x, PurePath):
return str(x)
return x
readwriter._to_seq = partial(readwriter._to_seq, converter=converter)
Or maybe more correct and full workaround would be to monkeypatch the reader/writer methods directly:
#wraps(readwriter.DataFrameWriter.parquet)
def parquet(self, path, mode=None, partitionBy=None, compression=None):
return parquet.__wrapped__(self, str(path), mode=mode,
partitionBy=partitionBy,
compression=compression)
readwriter.DataFrameWriter.parquet = parquet
When I run this code in Jupyter notebook it runs without any error but when I want to run it in VSCODE I get an error , It works by url and a token ,then I got the Json file and exported it to csv file to open in Excel :
This is the code ;
import json
import requests
import pandas
url = "url"
headers = {"Cookie":"token"}
response = requests.get(url, headers=headers)
data = response.json()
print(data)
a=data['tickers'][0]['items'][0]['days'][0]['items']
from copy import deepcopy
import pandas
def cross_join(left, right):
new_rows = []
for left_row in left:
for right_row in right:
temp_row = deepcopy(left_row)
for key, value in right_row.items():
temp_row[key] = value
new_rows.append(deepcopy(temp_row))
return new_rows
def flatten_list(data):
for elem in data:
if isinstance(elem, list):
yield from flatten_list(elem)
else:
yield elem
def json_to_dataframe(data_in):
def flatten_json(data, prev_heading=''):
if isinstance(data, dict):
rows = [{}]
for key, value in data.items():
rows = cross_join(rows, flatten_json(value, prev_heading + '.' + key))
elif isinstance(data, list):
rows = []
for i in range(len(data)):
[rows.append(elem) for elem in flatten_list(flatten_json(data[i], prev_heading))]
else:
rows = [{prev_heading[1:]: data}]
return rows
return pandas.DataFrame(flatten_json(data_in))
if __name__ == '__main__':
json_data = a
df = json_to_dataframe(json_data)
print(df)
df.to_excel("output.xlsx")
This is the Error ;
ModuleNotFoundError Traceback (most recent call last)
---> 59 df.to_excel("output.xlsx")
-> 2026 formatter.write(
--> 730 writer = ExcelWriter(stringify_path(writer), engine=engine)
---> 18 from openpyxl.workbook import Workbook
ModuleNotFoundError: No module named 'openpyxl'
What am I missing ?
Run this command from cmd window in VSCODE.
pip install openpyxl
I need to import different ipynb files, so I tried this:
https://jupyter-notebook.readthedocs.io/en/stable/examples/Notebook/Importing%20Notebooks.html
But I get no module named 'mynotebook' found. (I even tried it with other notebooks names, which definitely exist, but still not working)
Do you have any ideas about what I could do?
import io, os, sys, types
from IPython import get_ipython
from nbformat import read
from IPython.core.interactiveshell import InteractiveShell
def find_notebook(fullname, path=None):
name = fullname.rsplit('.', 1)[-1]
if not path:
path = ['']
for d in path:
nb_path = os.path.join(d, name + ".ipynb")
if os.path.isfile(nb_path):
return nb_path
# let import Notebook_Name find "Notebook Name.ipynb"
nb_path = nb_path.replace("_", " ")
if os.path.isfile(nb_path):
return nb_path
class NotebookLoader(object):
def __init__(self, path=None):
self.shell = InteractiveShell.instance()
self.path = path
def load_module(self, fullname):
"""import a notebook as a module"""
path = find_notebook(fullname, self.path)
print ("importing Jupyter notebook from %s" % path)
# load the notebook object
with io.open(path, 'r', encoding='utf-8') as f:
nb = read(f, 4)
# create the module and add it to sys.modules
# if name in sys.modules:
# return sys.modules[name]
mod = types.ModuleType(fullname)
mod.__file__ = path
mod.__loader__ = self
mod.__dict__['get_ipython'] = get_ipython
sys.modules[fullname] = mod
# extra work to ensure that magics that would affect the user_ns
# actually affect the notebook module's ns
save_user_ns = self.shell.user_ns
self.shell.user_ns = mod.__dict__
try:
for cell in nb.cells:
if cell.cell_type == 'code':
# transform the input to executable Python
code = self.shell.input_transformer_manager.transform_cell(cell.source)
# run the code in themodule
exec(code, mod.__dict__)
finally:
self.shell.user_ns = save_user_ns
return mod
class NotebookFinder(object):
def __init__(self):
self.loaders = {}
def find_module(self, fullname, path=None):
nb_path = find_notebook(fullname, path)
if not nb_path:
return
key = path
if path:
# lists aren't hashable
key = os.path.sep.join(path)
if key not in self.loaders:
self.loaders[key] = NotebookLoader(path)
return self.loaders[key]
sys.meta_path.append(NotebookFinder())
import mynotebook
I just want to import the code of another jupyter file
WOW, i also face this problem. I create a new env and after open jupyter, it can't find nbformat in my new installed env, so just:
pip install nbformat
If anyone actually reads this thanks!
Anyway on to the problem, every time I run my code I get an 'AttributeError' and I can't find where the issue is. I'm using Socket, tKinter, os and multiprocessing. Here's my code(I know its now the most pythony thing in the world but hey I'm just playing with sockets):
#---Import statments---#
import socket, os, multiprocessing
import tkinter as tk
#---global variables---#
setup = ''
cleintsocket = ''
#---Defs---#
def setup():
global host, port, user
host = setup_host_box.get()
port = setup_port_box.get()
user = setup_user_box.get()
def connect(self, hostname, connectingport):
self.connect((hostname, int(connectingport)))
print('connected')
multiprocessing.Process(target = resv()).start()
def create_sock(nhost, nport):
global cleintsocket
cleintsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connect(cleintsocket, nhost, nport)
def send(username, cleintsock):
'''to send a message'''
usrmsg = (username + ' - ' + chat_msg_box.get()).encode()
cleintsock.send(usrmsg)
def resv(sock):
'''resive subscript, run through mutiprosses module'''
while True:
rmsg = sock.recv(1024).decode()
chat_msg_display_text.insert('end.0.', rmsg)
def chat():
'''loads chat page'''
setup_host_text.pack_forget()
setup_host_box.pack_forget()
setup_port_text.pack_forget()
setup_port_box.pack_forget()
setup_user_text.pack_forget()
setup_user_box.pack_forget()
setup_confirm_button.pack_forget()
chat_msg_desplay_text.pack()
chat_msg_box.pack()
chat_msg_send_button.pack()
def start():
'''starts the setup page'''
setup_host_text.pack()
setup_host_box.pack()
setup_port_text.pack()
setup_port_box.pack()
setup_user_text.pack()
setup_user_box.pack()
setup_confirm_button.pack()
#---TK Setup---#
#--window setup--#
window = tk.Tk()
window.title('Chat')
window.geometry('600x600')
window.configure(background='#ffffff')
#--connection setup page--#
setup_host_text = tk.Label(window, text = 'Host')
setup_host_box = tk.Entry(window, bg = '#ffffff')
setup_port_text = tk.Label(window, text = 'Port')
setup_port_box = tk.Entry(window, bg = '#ffffff')
setup_user_text = tk.Label(window, text = 'Username')
setup_user_box = tk.Entry(window, bg = '#ffffff')
setup_confirm_button = tk.Button(window,text = 'Connect', command = setup())
#--chat page--#
chat_msg_box = tk.Entry(window, bg='#ffffff')
chat_msg_send_button = tk.Button(window, text = 'send', command = send(user, cleintsocket))
chat_msg_display_text = tk.Text(window, width=600, height=500, wrap = 'word')
#--------------#
start()
The python console is saying there is an error here chat_msg_send_button = tk.Button(window, text = 'send', command = send(user, cleintsocket)) that produces an AttributeError: 'str' object has no attribute 'send' error however I can't see any problems with it.
Please help.
Thanks again!
EDIT: Here's the error(Not needed now but this is for principle)
Traceback (most recent call last):
File ".../tkcleint.py", line 76, in <module>
chat_msg_send_button = tk.Button(window, text = 'send', command = send(user, cleintsocket))
File ".../tkcleint.py", line 29, in send
cleintsock.send(usrmsg)
AttributeError: 'str' object has no attribute 'send'
First off (as #R.Murry has pointed out) you are calling the functions immediately and passing their return value as the command which in this case is None, so I'd start by fixing that up:
setup_confirm_button = tk.Button(window,text = 'Connect', command = setup) #don't call setup here
...
def send_button_callback():
send(user, cleintsocket)
chat_msg_send_button = tk.Button(window, text = 'send', command = send_button_callback)
next, it is important to include the whole error message in your question because that is not the line that is running into problems:
Traceback (most recent call last):
File ".../test.py", line 76, in <module>
chat_msg_send_button = tk.Button(window, text = 'send', command = send(user, cleintsocket))
File ".../test.py", line 29, in send
cleintsock.send(usrmsg)
AttributeError: 'str' object has no attribute 'send'
You pass the variable cleintsocket into send and try to use the .send method of a socket, however it is initialized to an empty string:
cleintsocket = ''
so if you call send before it is changed to a socket you will get that error, simply check whether it has been initialized yet:
def send(username, cleintsock):
'''to send a message'''
if cleintsock: #not an empty string
usrmsg = (username + ' - ' + chat_msg_box.get()).encode()
cleintsock.send(usrmsg)
#else:window.bell() #play a error beep