Getting Import Error quite randomly when using plotly express and having multiple graphs on one page - callback

Relatively new to Dash, and this is a problem that has been vexing me for months now. I am making a multi-page app that shows some basic data trends using cards, and graphs embedded within cardbody. 30% of the time, the app works well without any errors and the other 70% it throws either one of the following:
ImportError: cannot import name 'ValidatorCache' from partially initialized module 'plotly.validator_cache' (most likely due to a circular import)
OR
ImportError: cannot import name 'Layout' from partially initialized module 'plotly.graph_objects' (most likely due to a circular import)
Both these appear quite randomly and I usually refresh the app to make them go away. But obviously I am doing something wrong. I have a set of dropdowns that trigger callbacks on graphs. I have been wracking my head about this. Any help/leads would be appreciated. The only pattern I see in the errors is they seem to emerge when the plotly express graphs are being called in the callbacks.
What am I doing wrong? I have searched all over online for help but nothing yet.
Sharing with some relevant snippets of code (this may be too long and many parts not important to the question, but to give you a general idea of what I have been working towards)
import dash
import dash_bootstrap_components as dbc
from dash.dependencies import Input, Output, State
import dash_core_components as dcc
import dash_html_components as html
import plotly.graph_objs as go
from plotly.subplots import make_subplots
import plotly.express as px
card_content1_1 = [
dbc.CardHeader([dbc.Row([html.H5("SALES VOLUME TREND", className = "font-weight-bold text-success"),
dbc.Button(
html.I(className="fa fa-window-maximize"),
color="success",
id="sales_maximize",
className="ml-auto",
# href="www.cogitaas.com"
)
])]),
dbc.CardBody(
[dcc.Graph(
id='sales_graph',
figure={},
style={'height':'30vh'}
# className="mt-5"
)])]
card_stacked_discount = [
dbc.CardHeader([dbc.Row([html.H5("VOLUMES UNDER DIFFERENT DISCOUNT LEVELS", className="font-weight-bold text-info text-center"),
dbc.Button(
html.I(className="fa fa-window-maximize"),
color="info",
id="discount_maximize",
className="ml-auto",
# href="www.cogitaas.com"
)
])]),
dbc.CardBody(
[dcc.Dropdown(
id = 'stacked_discount_dropdown',
options =stacked_discount_options,
value=stacked_discount_options[0].get('value'),
style={'color':'black'},
# multi=True
),
dcc.Graph(
id='stacked_discount_graph',
figure={},
style={'height':'30vh'}
)])]
cards = html.Div(
[
dbc.Row(
[
dbc.Col(dbc.Card(card_content1_1, color="success", outline=True,
style={'height':'auto'}), width=8),
],
className="mb-4",
),
dbc.Row(
[
dbc.Col(dbc.Card(card_stacked_discount, color="info", outline=True), width=8),
dbc.Col(dbc.Card([
dbc.Row([
dbc.Col(dbc.Card(disc_sub_title, color="info", inverse=True)),
]),
html.Br(),
dbc.Row([
dbc.Col(dbc.Card(disc_sub_card1, color="info", outline=True)),
]),
]), width=4)
],
className="mb-4",
),
]
)
tab1_content = dbc.Card(
dbc.CardBody(
[cards,]
),
className="mt-3",
)
tabs = dbc.Tabs(dbc.Tab(tab1_content, label="Data", label_style={'color':'blue'}, tab_style={"margin-left":"auto"}),])
content = html.Div([
html.Div([tabs]),
],id="page-content")
app.layout = html.Div([dcc.Location(id="url"), content])
#app.callback(
dash.dependencies.Output('sales_graph', 'figure'),
[dash.dependencies.Input('platform-dropdown', 'value'),
dash.dependencies.Input('signature-dropdown', 'value'),
dash.dependencies.Input('franchise-dropdown', 'value'),
dash.dependencies.Input('sales_maximize', 'n_clicks'),
dash.dependencies.Input('time-dropdown', 'value'),
])
def update_sales_graph(plat, sign, fran, maximize, time_per):
print(str(time_per)+"Test")
time_ax=[]
if isinstance(time_per,str):
time_ax.append(time_per)
time_per=time_ax
if (time_per==None) or ('Full Period' in (time_per)):
dff = df[(df.Platform==plat) & (df.Signature==sign) & (df.Franchise==fran)]
elif ('YTD' in time_per):
dff = df[(df.Platform == plat) & (df.Signature == sign) & (df.Franchise == fran) & (df.year==2020)]
else:
dff = df[(df.Platform==plat) & (df.Signature==sign) & (df.Franchise==fran) & (df.Qtr_yr.isin(time_per))]
fig = px.area(dff, x='Date', y='Qty_Orig', color_discrete_sequence=px.colors.qualitative.Dark2)
fig.add_trace(go.Scatter(x=dff['Date'], y=dff['Outliers'], mode = 'markers', name='Outliers',
line=dict(color='darkblue')))
fig.add_trace(go.Scatter(x=dff['Date'], y=dff['bestfit'], name='Long Term Trend',
line=dict(color='darkblue')))
fig.update_layout(font_family="Rockwell",
title={'text': fran + " Volume Trend",
'y': 0.99,
# 'x': 0.15,
# 'xanchor': 'auto',
'yanchor': 'top'
},
legend=dict(
orientation="h",
# y=-.15, yanchor="bottom", x=0.5, xanchor="center"
),
yaxis_visible=False, yaxis_showticklabels=False,
xaxis_title=None,
margin=dict(l=0, r=0, t=0, b=0, pad=0),
plot_bgcolor='White',
paper_bgcolor='White',
)
fig.update_xaxes(showgrid=False, zeroline=True)
fig.update_yaxes(showgrid=False, zeroline=True)
changed_id = [p['prop_id'] for p in dash.callback_context.triggered][0]
if 'maximize' in changed_id:
fig.show()
return fig
#app.callback(
dash.dependencies.Output('stacked_discount_graph', 'figure'),
[dash.dependencies.Input('platform-dropdown', 'value'),
dash.dependencies.Input('signature-dropdown', 'value'),
dash.dependencies.Input('franchise-dropdown', 'value'),
dash.dependencies.Input('discount_maximize', 'n_clicks'),
dash.dependencies.Input('stacked_discount_dropdown', 'value'),
dash.dependencies.Input('time-dropdown', 'value'),
])
def stacked_discount(plat, sign, fran, maximize, sales_days, time_per):
time_ax=[]
if isinstance(time_per,str):
time_ax.append(time_per)
time_per=time_ax
# else:
# time_per=list(time_per)
if (time_per==None) or ('Full Period' in (time_per)):
df_promo = df_promo_vol[(df_promo_vol.Platform==plat) & (df_promo_vol.Signature==sign) & (df_promo_vol.Franchise==fran)]
elif ('YTD' in time_per):
df_promo = df_promo_vol[(df_promo_vol.Platform == plat) & (df_promo_vol.Signature == sign) & (df_promo_vol.Franchise == fran) & (df_promo_vol.Year==2020)]
else:
df_promo = df_promo_vol[(df_promo_vol.Platform==plat) & (df_promo_vol.Signature==sign) & (df_promo_vol.Franchise==fran) & (df_promo_vol.Qtr_yr.isin(time_per))]
color_discrete_map = {
"0 - 10": "orange",
"10 - 15": "green",
"15 - 20": "blue",
"20 - 25": "goldenrod",
"25 - 30": "magenta",
"30 - 35": "red",
"35 - 40": "aqua",
"40 - 45": "violet",
"45 - 50": "brown",
"50 + ": "black"
}
category_orders = {'disc_range': ['0 - 10', '10 - 15', '15 - 20', '20 - 25', '25 - 30', '30 - 35', '35 - 40',
'40 - 45', '45 - 50', '50 + ']}
if (sales_days == None) or (sales_days == 'sales_act'):
fig = px.bar(df_promo, x='start', y='units_shipped', color='disc_range',
color_discrete_map=color_discrete_map,
category_orders=category_orders,
)
else:
fig = px.bar(df_promo, x='start', y='Date', color="disc_range",
color_discrete_map=color_discrete_map,
category_orders=category_orders,
)
fig.update_layout(font_family="Rockwell",
title={'text': fran + " Sales Decomposition",
'y': 0.99,
'x': 0.1,
# 'xanchor': 'auto',
'yanchor': 'top'
},
legend=dict(
orientation="h",
# y=-.15, yanchor="bottom", x=0.5, xanchor="center"
),
# yaxis_visible=False, yaxis_showticklabels=False,
xaxis_title=None,
margin=dict(l=0, r=0, t=30, b=30, pad=0),
plot_bgcolor='White',
paper_bgcolor='White',
)
fig.update_xaxes(showgrid=False, zeroline=True)
fig.update_yaxes(showgrid=False, zeroline=True)
changed_id = [p['prop_id'] for p in dash.callback_context.triggered][0]
if 'maximize' in changed_id:
fig.show()
return fig

Well, it appears I may have stumbled on to an answer. I was using the pretty much the same inputs for multiple callbacks and that could have been causing some interference with the sequencing of inputs. Once I integrated the code into one callback with multiple outputs, the problem seems to have disappeared.

Was dealing with this same issue where everything in my app worked fine, then I made an entirely separate section & callback that started throwing those circular import errors.
Was reluctant to re-arrange my (rightfully) separated callbacks to be just a single one and found you can fix the issue by just simply importing what the script says it's failing to get. In my case, plotly was trying to import the ValidatorCache and Layout so adding these to the top cleared the issue and now my app works as expected. Hope this helps someone experiencing a similar issue.
from plotly.graph_objects import Layout
from plotly.validator_cache import ValidatorCache

Related

How to interpret an XCOM within a task_group to keep a history of dynamically generated tasks within it?

I implemented a dag with a task_group that loops on the contents of a file.. This isn't ideal, I wish I could loop through the contents of an XCom, unless I'm mistaken, that isn't possible.
import json
import logging
from airflow.decorators import task_group
from airflow.operators.dummy import DummyOperator
from dtm.migration.helpers.tasks import (
check_compliance_of_objects_sizes,
)
from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator
from airflow.providers.google.cloud.sensors.gcs import (
GCSObjectExistenceSensor,
)
#task_group(group_id="copy_task_group")
def copy_and_verify(
inp_parameters_path: str,
impersonated_service_account: str,
):
try:
with open(inp_parameters_path, "r") as f:
inp_parameters = json.load(f)
for entry in inp_parameters:
sensor_source_file = GCSObjectExistenceSensor(
task_id=f"sensor_source_file_{entry}",
bucket=inp_parameters[entry]["source"]["bucket"],
object=(
f"{inp_parameters[entry]['source']['prefix']}"
f"/{inp_parameters[entry]['source']['object']}"
),
impersonation_chain=impersonated_service_account,
)
copy_file = GCSToGCSOperator(
task_id=f"copy_file_{entry}",
source_bucket=inp_parameters[entry]["source"]["bucket"],
source_object=(
f"{inp_parameters[entry]['source']['prefix']}"
f"/{inp_parameters[entry]['source']['object']}"
),
destination_bucket=inp_parameters[entry]["destination"][
"bucket"
],
destination_object=(
f"{inp_parameters[entry]['destination']['prefix']}"
f"/{inp_parameters[entry]['destination']['object']}"
),
impersonation_chain=impersonated_service_account,
)
check_size = check_compliance_of_objects_sizes(
date_of_execution="{{ds_nodash}}",
data_to_check=inp_parameters[entry],
impersonation_chain=impersonated_service_account,
)
sensor_destination_file = GCSObjectExistenceSensor(
task_id=f"sensor_destination_file_{entry}",
bucket=inp_parameters[entry]["destination"]["bucket"],
object=(
f"{inp_parameters[entry]['destination']['prefix']}"
f"/{inp_parameters[entry]['destination']['object']}"
),
impersonation_chain=impersonated_service_account,
)
end_op = DummyOperator(task_id=f"end_{entry}")
(
sensor_source_file
>> copy_file
>> check_size
>> sensor_destination_file
>> end_op
)
except FileNotFoundError:
logging.info(
f"File {inp_parameters_path} is generated in a prior task."
)
As you can see, I used try...except FileNotFoundError: because the error (file not found) is raised on the dag creation since the inp_parameters_path hadn't been created yet by the upstream tasks.
The inp_parameters_path is a file that is created upstream, but unfortunately, it seems that in order to display the logged steps of the task_group post execution I need to keep this file.. Which I don't because it will change from one day to another.
Example of removal of task_group contents after execution:
The contents were well present before the remove_mapping_file has run.
How to pass an XCom to a task_group the same way I can do it with a task? If that's not possible, how do I archive those created inp_parameters_path files per execution so that I can come back and browse the execution of the dag?
If it can help, here's how my dag operates :
#dag(
catchup=False,
schedule_interval="#daily",
max_active_runs=1,
dag_id="migrate_dtm_gcs_data_history",
start_date=datetime(2022, 12, 1),
dagrun_timeout=timedelta(minutes=20),
tags=[
"migration",
"dtm",
],
default_args=default_args,
)
def template_dag():
map = convert_csv_migration_to_map(
source_csv=f"{COMPOSER_GCS_LOC_PATH}/dags/dtm/migration/helpers/gcs_migration_mapping_archetype.csv",
delimiter=",",
)
updated_map_1 = update_mapping_with_gcp_project_and_buckets_ids(
src_mig_map=map,
env="dev",
impersonation_chain=IMPERSONATED_SERVICE_ACCOUNT,
)
list_files_and_prefixes = GCSListObjectsOperator(
task_id="list_files_and_prefixes",
bucket=LEGACY_DTM_HISTORY_BUCKET,
prefix="raw_data/datamart/{{ds_nodash}}",
impersonation_chain=IMPERSONATED_SERVICE_ACCOUNT,
)
updated_map_2 = match_data_with_migration_map(
src_mig_map=updated_map_1,
files_and_prefixes=list_files_and_prefixes.output,
)
list_of_maps = flatten_to_input_files_granularity(
src_mig_map=updated_map_2,
)
gcs_migration_maps_path = (
f"{COMPOSER_GCS_LOC_PATH}/dags/dtm/migration"
f"/helpers/gcs_migration_list_of_maps_{{ds_nodash}}.json"
)
write_list_of_maps_into_dict = gen_dict_file_from_list_of_map(
file_name=gcs_migration_maps_path,
list_of_maps=list_of_maps,
)
copy_task_group = copy_and_verify(
inp_parameters_path=gcs_migration_maps_path,
impersonated_service_account=IMPERSONATED_SERVICE_ACCOUNT,
)
remove_mapping_file = BashOperator(
task_id="remove_mapping_file",
bash_command=f"rm -vf {gcs_migration_maps_path}",
)
(
map
>> updated_map_1
>> list_files_and_prefixes
>> updated_map_2
>> list_of_maps
>> write_list_of_maps_into_dict
>> copy_task_group
>> remove_mapping_file
)
dag = template_dag()

CallBack Error updating graph.figure - DASH

New here and to the world of python.
Generating a histogram plot where it changes color when a postcode is selected.
Now I have the main histogram but when the postcode is selected from the dropdown it gives the error. Unable to figure out how to resolve this. I have provided my full code below.
Let me know if more info is required to figure out the error. Something to do with input
import pandas as pd
import plotly.express as px
from dash import Dash, html, dcc, Input, Output
app = Dash(__name__)
app.title = "EGMs in NSW"
# Load your data : br = .....
datalist=pd.read_excel('premises-list-Aug-2022.xlsx',skiprows=[0,1,2])
datalist['Licence type'] = datalist['Licence type'].str.split(" - ").str.get(1)
datalist['Licence type'] = datalist['Licence type'].str.title()
postcode = datalist['Postcode']
postcode = postcode.sort_values(ascending=True)
postcode=list(dict.fromkeys(postcode))
pd.Series(datalist['Postcode'].unique()).sort_values(ascending=True)
def make_histogram_plot():
# Create scatter plot
fig=px.histogram(
data_frame = datalist,
x='Licence type',y='EGMs',color='Postcode',title='EGMs by Licence Type in NSW',
histfunc='avg'
)
fig.update_traces(marker=dict(color="#002664"))
fig.update_layout(
plot_bgcolor='#EAEDF4',
xaxis_title=None,
yaxis_title='Average EGMs'
)
return fig
app.layout = html.Div(children=[
html.Div([
dcc.Dropdown(postcode,value=[], placeholder="Select postcodes", multi=True, id='postcodes')
]),
dcc.Graph(id = 'graph',figure = make_histogram_plot())
])
#app.callback(
Output('graph','figure'),
Input('postcodes','value')
)
def dropdown_changed(postcodes):
if postcodes==[]:
fig = make_histogram_plot()
else:
datalist2 = datalist[datalist['Postcode'].isin(postcodes)],
# Create scatter plot
fig=px.histogram(
data_frame = datalist2,
x='Licence type',y='EGMs',color='Postcode',title='EGMs by Licence Type in NSW',
histfunc='avg'
)
fig.update_traces(marker=dict(color="#002664"))
fig.update_layout(
plot_bgcolor='#EAEDF4',
xaxis_title=None,
yaxis_title='Average EGMs'
)
return fig
# Start the server
if __name__ == '__main__':
app.run_server(debug=True)

Understanding layout rules in PySimpleGUI

I am trying to build a GUI with PySimpleGUI for the first time and am struggling with the layout reuse rules.
The overall aim of the script will be to make entering data into a CSV table easier for users. For this purpose, the users will be able to enter data into a form, submit the data, and decide if they want to add another data set or exit.
I have tried to work with nested functions but am constantly breaking the script because I am supposed to use a new layout for each window. I have defined 3 different windows with their own layouts so far:
window1 = main window where data can be added
window2 = window with error notification
window3 = window opening after SUBMIT, asking users if they want to continue
In addition, I am trying to call the function for window1 again if users decide to add more data (repeatGUI()), but this is not permitted.
I am aware that there are other issues in the script, too, but I would mainly appreciate some input on the layout-reuse issue. What is the proper way of opening an input window multiple times?
DRAFT SCRIPT:
import csv
import PySimpleGUI as sg
sg.main_get_debug_data()
myfile="C:\\######\\File.csv"
counter=0
def buildGUI(): # defining GUI frame, title, fields and buttons
sg.ChangeLookAndFeel('GreenTan')
window1 = sg.FlexForm('MY DB', size = (900,700), resizable=True)
layout1 = [
[sg.Text('Please enter a new factoid:')],
[sg.Text('Factoid ID', size=(15, 1), key="-FACTOID-")],
[sg.Text('Person ID', size=(15, 1), key="-PERSON-")],
[sg.Text('Exact Date', size=(10, 1)), sg.InputText()],
[sg.Text('Event Before', size=(10, 1)), sg.InputText()],
[sg.Text('Event After', size=(10, 1)), sg.InputText()],
[sg.Text('Event Start', size=(10, 1)), sg.InputText()],
[sg.Text('Event End', size=(10, 1)), sg.InputText()],
[sg.Text('Event Type', size=(15, 1)), sg.InputText()],
[sg.Text('Person Name', size=(15, 1)), sg.InputText()],
[sg.Text('Person Title', size=(15, 1)), sg.InputText()],
[sg.Text('Person Function', size=(15, 1)), sg.InputText()],
[sg.Text('Place Name', size=(15, 1)), sg.InputText()],
[sg.Text('Institution Name', size=(15, 1)), sg.InputText()],
[sg.Text('Related Persons', size=(15, 1)), sg.InputText()],
[sg.Text('Alternative Names', size=(15, 1)), sg.InputText()],
[sg.Text('Source Quote', size=(10, 1)), sg.InputText()],
[sg.Text('Additional Info', size=(10, 1)), sg.InputText()],
[sg.Text('Comment', size=(10, 1)), sg.InputText()],
[sg.Text('Source', size=(10, 1)), sg.InputText()],
[sg.Text('Source URL', size=(10, 1)), sg.InputText()],
[sg.Text('Info Dump', size=(10, 1)), sg.InputText()],
[sg.Text(size=(70,1), key='-MESSAGE1-')],
[sg.Submit(), sg.Button('Clear Input')]
]
while True: # The Event Loop
event1, values1 = window1.Layout(layout1).Read()
print(layout1)
def startGUI(event1, values1): # start GUI and get data
# interact with user to get input
if event1 == 'Submit': # user clicked "Submit"
def submitGUI(values1): # submitting new data via GUI
fact_id=("#") # place holder: to be calculated later
pers_id=("#") # place holder: to be calculated later
entries=[fact_id, pers_id]
for v in values1.values():
entries.append(v)
try:
with open(myfile, 'a', encoding="utf-8") as f:
w=csv.writer(f)
w.writerow(entries) # write list items to CSV file
f.close()
try:
window3 = sg.FlexForm('NEW DATA?', size = (500,100))
layout3 = [
[sg.Text("Your input has been saved! Do you want to add a new factoid?")],
[sg.Text(size=(70,1), key='-MESSAGE2-')],
[sg.Button("YES"), sg.Button("NO"), sg.Exit()]
]
while True:
event3, values3 = window3.Layout(layout3).Read()
if event3 == 'YES': # user clicked YES
window1.close()
try:
repeatGUI() # this breaks layout rules!
except:
print("Not allowed!")
pass
elif event3 == 'NO':
window3['-MESSAGE2-'].update("See you again soon!")
window1.close()
elif event3 in (sg.WINDOW_CLOSE_ATTEMPTED_EVENT, 'Exit'):
window3.close()
except:
pass
except PermissionError:
window2 = sg.FlexForm('DB ERROR', size = (500,100))
layout2 = [
[sg.Text("Someone else is using the file! Please try again later.")],
[sg.Exit()]
]
event2, values2 = window2.Layout(layout2).Read()
if event2 in (sg.WINDOW_CLOSE_ATTEMPTED_EVENT, 'Exit'):
window2.close() # user clicked "Exit"
submitGUI(values1)
elif event1 == 'Clear Input': # user clicked "Cancel"
window1.Refresh()
elif event1 == sg.WINDOW_CLOSE_ATTEMPTED_EVENT: # user closed window
window1.close() # AttributeError: module 'PySimpleGUI' has no attribute 'WIN_CLOSED'
startGUI(event1, values1)
buildGUI()
def repeatGUI():
counter+=1
print(counter)
buildGUI()
Following way show that you use the variable layout1 as layout of sg.Window again and again, none of element in layout1 initialized after 1st time using.
while True: # The Event Loop
event1, values1 = window1.Layout(layout1).Read()
...
Following code preferred,
window1 = sg.Window('Title1', layout1, finalize=True)
while True:
event1, values1 = window1.read()
...
or
def make_window1():
layout = [
[sg.Text('Please enter a new factoid:')],
....
]
return sg.Window('Title1', layout, finalized=True)
window1 = make_window1()
while True:
event1, values1 = window1.read()
...
Based on #jason-yang 's answer, I was able to fix my layout issue and revised the whole script to
a) define layouts outside the function that starts the GUI
and
b) get rid of the attempt to open the same input window several times.
Having a persistent window is much more suitable in my case, so I am using a "Clear" button to allow new data input:
elif event1 == 'Clear to add new data': # user clicked "Clear"
window1['-MESSAGE1-'].update("Ready for your next data set!")
# update all input fields
window1['-DATE-'].update('')
window1['-BEFORE-'].update('')
window1['-AFTER-'].update('')
window1['-START-'].update('')
window1['-END-'].update('')
window1['-EVENT-'].update('')
window1['-PERSNAME-'].update('')

How can I export Jira issues to BitBucket

Ive just moved my projects code from java.net to BitBucket. But my jira issue tracking is still hosted on java.net, although BitBucket does have some options for linking to an external issue tracker I don't think I can use it for java.net, not least because I do not have the admin priviledges need to install the DVCS connector.
So I thought an alternative option would be to export and then import the issues into BitBucket issue tracker, is that possible ?
Progress so far
So I tried following the steps in both informative answers using OSX below but I hit a problem - I'm rather confused about what the script would actually be called because in the answers it talks about export.py but no such script exists with that name so I renamed the one I downloaded.
sudo easy_install pip (OSX)
pip install jira
pip install configparser
easy_install -U setuptools
Go to https://bitbucket.org/reece/rcore, select downloads tab, download zip and unzip, and rename to reece ( for some reason git clone https://bitbucket.org/reece/rcore fails with error)
cd reece/rcore
Save script as export.py in rcore subfolder
Replace iteritems with items in import.py
Replace iteritems with types/immutabledict.py
Create .config in rcore folder
Create .config/jira-issues-move-to-bitbucket.conf containing
jira-username=paultaylor
jira-hostname=https://java.net/jira/browse/JAUDIOTAGGER
jira-password=password
Run python export.py --jira-project jaudiotagger
gives
macbook:rcore paul$ python export.py --jira-project jaudiotagger
Traceback (most recent call last):
File "export.py", line 24, in <module>
import configparser
ImportError: No module named configparser
- Run python export.py --jira-project jaudiotagger
I need to run pip insdtall as root so did
sudo pip install configparser
and that worked
but now
python export.py --jira.project jaudiotagger
gives
File "export.py" line 35, in <module?
from jira.client import JIRA
ImportError: No module named jira.client
You can import issues into BitBucket, they just need to be in the appropriate format. Fortunately, Reece Hart has already written a Python script to connect to a Jira instance and export the issues.
To get the script to run I had to install the Jira Python package as well as the latest version of rcore (if you use pip you get an incompatible previous version, so you have to get the source). I also had to replace all instances of iteritems with items in the script and in rcore/types/immutabledict.py to make it work with Python 3. You will also need to fill in the dictionaries (priority_map, person_map, etc) with the values your project uses. Finally, you need a config file to exist with the connection info (see comments at the top of the script).
The basic command line usage is export.py --jira-project <project>
Once you've got the data exported, see the instructions for importing issues to BitBucket
#!/usr/bin/env python
"""extract issues from JIRA and export to a bitbucket archive
See:
https://confluence.atlassian.com/pages/viewpage.action?pageId=330796872
https://confluence.atlassian.com/display/BITBUCKET/Mark+up+comments
https://bitbucket.org/tutorials/markdowndemo/overview
2014-04-12 08:26 Reece Hart <reecehart#gmail.com>
Requires a file ~/.config/jira-issues-move-to-bitbucket.conf
with content like
[default]
jira-username=some.user
jira-hostname=somewhere.jira.com
jira-password=ur$pass
"""
import argparse
import collections
import configparser
import glob
import itertools
import json
import logging
import os
import pprint
import re
import sys
import zipfile
from jira.client import JIRA
from rcore.types.immutabledict import ImmutableDict
priority_map = {
'Critical (P1)': 'critical',
'Major (P2)': 'major',
'Minor (P3)': 'minor',
'Nice (P4)': 'trivial',
}
person_map = {
'reece.hart': 'reece',
# etc
}
issuetype_map = {
'Improvement': 'enhancement',
'New Feature': 'enhancement',
'Bug': 'bug',
'Technical task': 'task',
'Task': 'task',
}
status_map = {
'Closed': 'resolved',
'Duplicate': 'duplicate',
'In Progress': 'open',
'Open': 'new',
'Reopened': 'open',
'Resolved': 'resolved',
}
def parse_args(argv):
def sep_and_flatten(l):
# split comma-sep elements and flatten list
# e.g., ['a','b','c,d'] -> set('a','b','c','d')
return list( itertools.chain.from_iterable(e.split(',') for e in l) )
cf = configparser.ConfigParser()
cf.readfp(open(os.path.expanduser('~/.config/jira-issues-move-to-bitbucket.conf'),'r'))
ap = argparse.ArgumentParser(
description = __doc__
)
ap.add_argument(
'--jira-hostname', '-H',
default = cf.get('default','jira-hostname',fallback=None),
help = 'host name of Jira instances (used for url like https://hostname/, e.g., "instancename.jira.com")',
)
ap.add_argument(
'--jira-username', '-u',
default = cf.get('default','jira-username',fallback=None),
)
ap.add_argument(
'--jira-password', '-p',
default = cf.get('default','jira-password',fallback=None),
)
ap.add_argument(
'--jira-project', '-j',
required = True,
help = 'project key (e.g., JRA)',
)
ap.add_argument(
'--jira-issues', '-i',
action = 'append',
default = [],
help = 'issue id (e.g., JRA-9); multiple and comma-separated okay; default = all in project',
)
ap.add_argument(
'--jira-issues-file', '-I',
help = 'file containing issue ids (e.g., JRA-9)'
)
ap.add_argument(
'--jira-components', '-c',
action = 'append',
default = [],
help = 'components criterion; multiple and comma-separated okay; default = all in project',
)
ap.add_argument(
'--existing', '-e',
action = 'store_true',
default = False,
help = 'read existing archive (from export) and merge new issues'
)
opts = ap.parse_args(argv)
opts.jira_components = sep_and_flatten(opts.jira_components)
opts.jira_issues = sep_and_flatten(opts.jira_issues)
return opts
def link(url,text=None):
return "[{text}]({url})".format(url=url,text=url if text is None else text)
def reformat_to_markdown(desc):
def _indent4(mo):
i = " "
return i + mo.group(1).replace("\n",i)
def _repl_mention(mo):
return "#" + person_map[mo.group(1)]
#desc = desc.replace("\r","")
desc = re.sub("{noformat}(.+?){noformat}",_indent4,desc,flags=re.DOTALL+re.MULTILINE)
desc = re.sub(opts.jira_project+r"-(\d+)",r"issue #\1",desc)
desc = re.sub(r"\[~([^]]+)\]",_repl_mention,desc)
return desc
def fetch_issues(opts,jcl):
jql = [ 'project = ' + opts.jira_project ]
if opts.jira_components:
jql += [ ' OR '.join([ 'component = '+c for c in opts.jira_components ]) ]
if opts.jira_issues:
jql += [ ' OR '.join([ 'issue = '+i for i in opts.jira_issues ]) ]
jql_str = ' AND '.join(["("+q+")" for q in jql])
logging.info('executing query ' + jql_str)
return jcl.search_issues(jql_str,maxResults=500)
def jira_issue_to_bb_issue(opts,jcl,ji):
"""convert a jira issue to a dictionary with values appropriate for
POSTing as a bitbucket issue"""
logger = logging.getLogger(__name__)
content = reformat_to_markdown(ji.fields.description) if ji.fields.description else ''
if ji.fields.assignee is None:
resp = None
else:
resp = person_map[ji.fields.assignee.name]
reporter = person_map[ji.fields.reporter.name]
jiw = jcl.watchers(ji.key)
watchers = [ person_map[u.name] for u in jiw.watchers ] if jiw else []
milestone = None
if ji.fields.fixVersions:
vnames = [ v.name for v in ji.fields.fixVersions ]
milestone = vnames[0]
if len(vnames) > 1:
logger.warn("{ji.key}: bitbucket issues may have only 1 milestone (JIRA fixVersion); using only first ({f}) and ignoring rest ({r})".format(
ji=ji, f=milestone, r=",".join(vnames[1:])))
issue_id = extract_issue_number(ji.key)
bbi = {
'status': status_map[ji.fields.status.name],
'priority': priority_map[ji.fields.priority.name],
'kind': issuetype_map[ji.fields.issuetype.name],
'content_updated_on': ji.fields.created,
'voters': [],
'title': ji.fields.summary,
'reporter': reporter,
'component': None,
'watchers': watchers,
'content': content,
'assignee': resp,
'created_on': ji.fields.created,
'version': None, # ?
'edited_on': None,
'milestone': milestone,
'updated_on': ji.fields.updated,
'id': issue_id,
}
return bbi
def jira_comment_to_bb_comment(opts,jcl,jc):
bbc = {
'content': reformat_to_markdown(jc.body),
'created_on': jc.created,
'id': int(jc.id),
'updated_on': jc.updated,
'user': person_map[jc.author.name],
}
return bbc
def extract_issue_number(jira_issue_key):
return int(jira_issue_key.split('-')[-1])
def jira_key_to_bb_issue_tag(jira_issue_key):
return 'issue #' + str(extract_issue_number(jira_issue_key))
def jira_link_text(jk):
return link("https://invitae.jira.com/browse/"+jk,jk) + " (Invitae access required)"
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
opts = parse_args(sys.argv[1:])
dir_name = opts.jira_project
if opts.jira_components:
dir_name += '-' + ','.join(opts.jira_components)
if opts.jira_issues_file:
issues = [i.strip() for i in open(opts.jira_issues_file,'r')]
logger.info("added {n} issues from {opts.jira_issues_file} to issues list".format(n=len(issues),opts=opts))
opts.jira_issues += issues
opts.dir = os.path.join('/','tmp',dir_name)
opts.att_rel_dir = 'attachments'
opts.att_abs_dir = os.path.join(opts.dir,opts.att_rel_dir)
opts.json_fn = os.path.join(opts.dir,'db-1.0.json')
if not os.path.isdir(opts.att_abs_dir):
os.makedirs(opts.att_abs_dir)
opts.jira_issues = list(set(opts.jira_issues)) # distinctify
jcl = JIRA({'server': 'https://{opts.jira_hostname}/'.format(opts=opts)},
basic_auth=(opts.jira_username,opts.jira_password))
if opts.existing:
issues_db = json.load(open(opts.json_fn,'r'))
existing_ids = [ i['id'] for i in issues_db['issues'] ]
logger.info("read {n} issues from {fn}".format(n=len(existing_ids),fn=opts.json_fn))
else:
issues_db = dict()
issues_db['meta'] = {
'default_milestone': None,
'default_assignee': None,
'default_kind': "bug",
'default_component': None,
'default_version': None,
}
issues_db['attachments'] = []
issues_db['comments'] = []
issues_db['issues'] = []
issues_db['logs'] = []
issues_db['components'] = [ {'name':v.name} for v in jcl.project_components(opts.jira_project) ]
issues_db['milestones'] = [ {'name':v.name} for v in jcl.project_versions(opts.jira_project) ]
issues_db['versions'] = issues_db['milestones']
# bb_issue_map: bb issue # -> bitbucket issue
bb_issue_map = ImmutableDict( (i['id'],i) for i in issues_db['issues'] )
# jk_issue_map: jira key -> bitbucket issue
# contains only items migrated from JIRA (i.e., not preexisting issues with --existing)
jk_issue_map = ImmutableDict()
# issue_links is a dict of dicts of lists, using JIRA keys
# e.g., links['CORE-135']['depends on'] = ['CORE-137']
issue_links = collections.defaultdict(lambda: collections.defaultdict(lambda: []))
issues = fetch_issues(opts,jcl)
logger.info("fetch {n} issues from JIRA".format(n=len(issues)))
for ji in issues:
# Pfft. Need to fetch the issue again due to bug in JIRA.
# See https://bitbucket.org/bspeakmon/jira-python/issue/47/, comment on 2013-10-01 by ssonic
ji = jcl.issue(ji.key,expand="attachments,comments")
# create the issue
bbi = jira_issue_to_bb_issue(opts,jcl,ji)
issues_db['issues'] += [bbi]
bb_issue_map[bbi['id']] = bbi
jk_issue_map[ji.key] = bbi
issue_links[ji.key]['imported from'] = [jira_link_text(ji.key)]
# add comments
for jc in ji.fields.comment.comments:
bbc = jira_comment_to_bb_comment(opts,jcl,jc)
bbc['issue'] = bbi['id']
issues_db['comments'] += [bbc]
# add attachments
for ja in ji.fields.attachment:
att_rel_path = os.path.join(opts.att_rel_dir,ja.id)
att_abs_path = os.path.join(opts.att_abs_dir,ja.id)
if not os.path.exists(att_abs_path):
open(att_abs_path,'w').write(ja.get())
logger.info("Wrote {att_abs_path}".format(att_abs_path=att_abs_path))
bba = {
"path": att_rel_path,
"issue": bbi['id'],
"user": person_map[ja.author.name],
"filename": ja.filename,
}
issues_db['attachments'] += [bba]
# parent-child is task-subtask
if hasattr(ji.fields,'parent'):
issue_links[ji.fields.parent.key]['subtasks'].append(jira_key_to_bb_issue_tag(ji.key))
issue_links[ji.key]['parent task'].append(jira_key_to_bb_issue_tag(ji.fields.parent.key))
# add links
for il in ji.fields.issuelinks:
if hasattr(il,'outwardIssue'):
issue_links[ji.key][il.type.outward].append(jira_key_to_bb_issue_tag(il.outwardIssue.key))
elif hasattr(il,'inwardIssue'):
issue_links[ji.key][il.type.inward].append(jira_key_to_bb_issue_tag(il.inwardIssue.key))
logger.info("migrated issue {ji.key}: {ji.fields.summary} ({components})".format(
ji=ji,components=','.join(c.name for c in ji.fields.components)))
# append links section to content
# this section shows both task-subtask and "issue link" relationships
for src,dstlinks in issue_links.iteritems():
if src not in jk_issue_map:
logger.warn("issue {src}, with issue_links, not in jk_issue_map; skipping".format(src=src))
continue
links_block = "Links\n=====\n"
for desc,dsts in sorted(dstlinks.iteritems()):
links_block += "* **{desc}**: {links} \n".format(desc=desc,links=", ".join(dsts))
if jk_issue_map[src]['content']:
jk_issue_map[src]['content'] += "\n\n" + links_block
else:
jk_issue_map[src]['content'] = links_block
id_counts = collections.Counter(i['id'] for i in issues_db['issues'])
dupes = [ k for k,cnt in id_counts.iteritems() if cnt>1 ]
if dupes:
raise RuntimeError("{n} issue ids appear more than once from existing {opts.json_fn}".format(
n=len(dupes),opts=opts))
json.dump(issues_db,open(opts.json_fn,'w'))
logger.info("wrote {n} issues to {opts.json_fn}".format(n=len(id_counts),opts=opts))
# write zipfile
os.chdir(opts.dir)
with zipfile.ZipFile(opts.dir + '.zip','w') as zf:
for fn in ['db-1.0.json']+glob.glob('attachments/*'):
zf.write(fn)
logger.info("added {fn} to archive".format(fn=fn))
NOTE: I'm writing a new answer because writing this in a comment would be horrible, but most of the credit goes to #Turch's answer.
My steps (in OSX and Debian machines, both worked fine):
apt-get install python-pip (Debian) or sudo easy_install pip (OSX)
pip install jira
pip install configparser
easy_install -U setuptools (not sure if really needed)
Download or clone the source code from https://bitbucket.org/reece/rcore/ in your home folder, for example. Note: don't download using pip, it will get the 0.0.2 version and you need the 0.0.3.
Download the Python script created by Reece, mentioned by #Turch, and place it inside of the rcore folder.
Follow the instructions by #Turch: I also had to replace all instances of iteritems with items in the script and in rcore/types/immutabledict.py to make it work with Python 3. You will also need to fill in the dictionaries (priority_map, person_map, etc) with the values your project uses. Finally, you need a config file to exist with the connection info (see comments at the top of the script). Note: I used hostname like jira.domain.com (no http or https).
(This change did the trick for me) I had to change part of the line 250 from 'https://{opts.jira_hostname}/' to 'http://{opts.jira_hostname}/'
To finish, run the script like #Turch mentioned: The basic command line usage is export.py --jira-project <project>
The file was placed in /tmp/.zip for me.
The file was perfectly accepted in the BitBucket importer today.
Hooray for Reece and Turch! Thanks guys!

D3 Multi Line Graph with Dots

I am new to D3.js. I love it but I am having real trouble figuring out the best approach to structuring data.
I would ideally like to create a simple multiline graph that has over points over the selected points. Firstly I have the multiple lines created but trying to add the points has stumped me, and I think it has to do with the structure of my data.
Here is my working fiddle. I'm not sure if I should be trying to use d3.nest to re-arrange the data
I have a json object that I am retrieving from a google form which is all nice and smooth. This is what it looks like:
var data = [{
"description": "Global warming is a serious and pressing problem. We should begin taking steps now even if this involves significant costs",
"year2013": 40,
"year2012": 36,
"year2011": 41,
"year2010": 46,
"year2009": 48,
"year2008": 60,
"year2006": 68,
}, {
"description": "The problem of global warming should be addressed, but its effects will be gradual, so we can deal with the problem gradually by taking steps that are low in cost",
"year2013": 44,
"year2012": 45,
"year2011": 40,
"year2010": 40,
"year2009": 39,
"year2008": 32,
"year2006": 24,
}, {
"description": "Until we are sure that global warming is really a problem, we should not take any steps that would have economic costs",
"year2013": 16,
"year2012": 18,
"year2011": 19,
"year2010": 13,
"year2009": 13,
"year2008": 8,
"year2006": 7,
}, {
"description": "Don't know / refused",
"year2013": 1,
"year2012": 1,
"year2011": 1,
"year2010": 1,
"year2009": 1,
"year2008": 0,
"year2006": 1,
}]
Any help would be appreciated, I have been at it for days.
Cheers!
First - I would flatten your data
data = [
{date:"2011",type: "line0", amount:20}
...
]
Then nest your data by type
nested = d3.nest()
.key( (d) -> return d.type )
.entries(data)
Then append your line groups
# Line Groups
groups = container.selectAll('g.full-line')
.data(nested, (d) -> return d.key )
# ENTER
groups.enter().append('svg:g')
.attr( 'class', (d,i) -> "full-line#{i}" )
# EXIT
d3.transition(groups.exit()).remove()
# TRANSITION
d3.transition(groups)
Then append your chart lines
# Individual Lines
lines = groups.selectAll('.line').data (d)-> [d.values]
# ENTER
lines.enter().append("svg:path")
.attr("class","line")
.attr("d", d3.svg.line()
.interpolate(interpolate)
.defined(defined)
.x( (d,i) -> return xScale(d,i) )
.y( (d,i) -> return yScale(d,i) ) )
# EXIT
d3.transition( groups.exit().selectAll('.line') )
.attr("d",
d3.svg.line()
.interpolate(interpolate)
.defined(defined)
.x( (d,i) -> return xScale(d,i) )
.y( (d,i) -> return yScale(d,i) ) )
# TRANSITION
d3.transition(lines)
.attr("d",
d3.svg.line()
.interpolate(interpolate)
.defined(defined)
.x( (d,i) -> return xScale(d,i) )
.y( (d,i) -> return yScale(d,i) ) )
Thanks
I ended up using something similar.
/* Transform Data */
data = data.map(function (d) {
return {
country: d.country,
date: new Date(d.year.toString()),
value: d.value
};
});
/* Nest Data */
data = d3.nest().key(function (d) {
return d.country;
}).entries(data);`