i use these packages:
openpyxl - copy excel templates, opened copied file, save data from db, print it and then delete;
pywin32 - send for printing to remote network printer by network name;
after some processes, i don't know which - excel process window still opened. (attach screenshot)
i attach the most using in my project functions.
this program it's like a web service, which listening 5000 port and print in needed template.
i delete all created files, because of no need to store all of them.
from openpyxl import load_workbook
import os
import app_config as config
import printers.printers as p
from datetime import datetime
import shutil
import time
class EditExcelTemplate:
def __init__(self, template_name):
now = datetime.now()
report_name = "_{}{}{}_{}{}{}_{}".format(now.year, now.month, now.day,
now.hour, now.minute, now.second,
now.microsecond)
self.report_path = config.EXCEL_REPORT_PATH.format(template_name +
report_name)
shutil.copy(src=config.EXCEL_TEMPLATE_PATH.format(template_name),
dst=self.report_path)
# self.wb = load_workbook(filename=config.EXCEL_TEMPLATE_PATH.format(template_name))
start_load = time.time()
self.wb = load_workbook(filename=self.report_path,
keep_links=False,
keep_vba=False,
data_only=True)
end_load = time.time()
print('LOAD WORKBOOK|{}'.format(str(end_load - start_load)))
self.ws = self.wb.active
self.answer = {'file_name': template_name.upper()}
def write_workbook(self, row_dest, column_dest, value):
c = self.ws.cell(row=row_dest, column=column_dest)
c.value = value
def save_excel(self):
self.wb.save(self.report_path)
def print_excel(self, printer_no):
p.print_excel(printer_no=printer_no, path_to_file=self.report_path)
def print_excel_file(self, printer_name):
p.print_excel_file(printer_name=printer_name, path_to_file=self.report_path)
import win32api
import app_config
import os, time
def print_excel(printer_no, path_to_file):
printer_name = app_config.PRINTER_NAMES[printer_no]
win32api.ShellExecute(
1,
'printto',
path_to_file,
'{}'.format(printer_name),
'.',
0
)
def delete_file(path_to_file, try_count=1):
if os.path.exists(path=path_to_file):
file_name = path_to_file.split('\\')[-1]
while try_count < 60:
try:
os.remove(path_to_file)
print('File {} deleted!'.format(file_name))
break
except PermissionError:
print('Can not delete file {}. Hold {} sec.'.format(file_name, try_count))
time.sleep(1.0)
try_count += 1
Related
In Chrome, if you close the pop up dialog of this page, then it won't show if you open the page again. However, in my following code, the pop up dialog still shows in a second run even you close it in the first run, it seems to be a issue with the persistent storage, but I don't know how to solve the issue, any help?
from PyQt6.QtCore import *
from PyQt6.QtCore import pyqtSlot as Slot
from PyQt6.QtCore import pyqtSignal as Signal
from PyQt6.QtGui import *
from PyQt6.QtWidgets import *
from PyQt6.QtWebEngineWidgets import *
from PyQt6.QtWebEngineCore import *
import sys
import os
class WebEngineView(QWebEngineView): # QWebEngineView
def __init__(self, parent=None):
super().__init__(parent)
self.webpage = QWebEnginePage()
# self.webpage.javaScriptConsoleMessage = lambda level, message, lineNumber, sourceID: print(message) # if level > QWebEnginePage.WarningMessageLevel else None # 关闭js console msg,不起作用→self.javaScriptConsoleMessage = None;https://doc.qt.io/qt-5/qwebenginepage.html#JavaScriptConsoleMessageLevel-enum
self.setPage(self.webpage)
self.webpage.load(QUrl('https://fanyi.baidu.com/'))
if __name__ == "__main__":
app = QApplication(sys.argv)
webEngineView = WebEngineView()
webEngineView.showMaximized()
sys.exit(app.exec())
My solotion:
from PyQt6.QtCore import *
from PyQt6.QtCore import pyqtSlot as Slot
from PyQt6.QtCore import pyqtSignal as Signal
from PyQt6.QtGui import *
from PyQt6.QtWidgets import *
from PyQt6.QtWebEngineWidgets import *
from PyQt6.QtWebEngineCore import *
import sys
import os
class WebEnginePage(QWebEnginePage): # QWebEngineView
def __init__(self, profile, parent=None):
super().__init__(profile, parent)
def contextMenuEvent(self, event): # 算是一种折中的方法,因为其他的方法好像因为bug的原因不起作用
self.menu = self.createStandardContextMenu() # page().
selectedText = self.selectedText()
if selectedText:
self.menu.addSeparator()
self.menu.addAction('谷歌搜索', lambda: QDesktopServices.openUrl(QUrl(f'https://www.google.com/search?q={selectedText}')))
self.menu.popup(event.globalPos()) # 这种貌似怪异的做法也不能改成show或pos;using menu.exec() might lead to consolle warnings and painting artifacts, so using popup() is better
class WebEngineView(QWebEngineView): # QWebEngineView
def __init__(self, parent=None):
super().__init__(parent)
self.webEngineProfile = QWebEngineProfile('EngkudictWebEngineProfile ')
# self.webEngineProfile.setPersistentCookiesPolicy(QWebEngineProfile.PersistentCookiesPolicy.ForcePersistentCookies)
print(self.webEngineProfile.persistentCookiesPolicy(), self.webEngineProfile.isOffTheRecord())
self.webpage = WebEnginePage(self.webEngineProfile) # QWebEnginePage(self.webEngineProfile)
# self.webpage.destroyed.connect(lambda obj: self.webEngineProfile.deleteLater()) #这种方式不行 If the profile is not the default profile, the caller must ensure that the profile stays alive for as long as the page does.
self.setPage(self.webpage)
self.webpage.load(QUrl('https://fanyi.baidu.com/'))
# webEngineProfile = self.page().profile()
# # webEngineProfile.setPersistentCookiesPolicy(QWebEngineProfile.PersistentCookiesPolicy.ForcePersistentCookies)
# print(webEngineProfile.persistentCookiesPolicy(), webEngineProfile.isOffTheRecord(), webEngineProfile.persistentStoragePath()) # Qt6 PersistentCookiesPolicy.NoPersistentCookies True=====Qt6 1 False
# # self.load(QUrl('https://fanyi.baidu.com/'))
# self.load(QUrl('https://doc.qt.io/qt-6/qwebengineprofile.html#QWebEngineProfile-1'))
#Slot(QCloseEvent)
def closeEvent(self, event):
self.setPage(None) # To avoid msg: Release of profile requested but WebEnginePage still not deleted. Expect troubles ! https://github.com/qutebrowser/qutebrowser/commit/e6ae8797e71a678bef97a13b9057e29442e0ef48
# del self.webEngineProfile
# self.webEngineProfile.deleteLater() # A disk-based QWebEngineProfile should be destroyed on or before application exit, otherwise the cache and persistent data may not be fully flushed to disk. https://doc.qt.io/qt-6/qwebengineprofile.html#QWebEngineProfile-1
if __name__ == "__main__":
# QGuiApplication.setAttribute(Qt.AA_EnableHighDpiScaling) # 任务Qt6 不需要了Qt High DPI scaling is now activated by default; the default rounding policy is PassThrough
# os.putenv("QT_ENABLE_HIGHDPI_SCALING", '1')
os.putenv("QT_SCALE_FACTOR", '1.6')
app = QApplication(sys.argv)
webEngineView = WebEngineView()
webEngineView.showMaximized()
sys.exit(app.exec())
Files to upload will be like WFSIV0101202001.318.tar.gz,WFSIV0101202001.2624.tar.gz etc.
INPUT_FILE_PATH = 'C:\Files to upload'
try:
import os
from google.cloud import storage
import sys
import pymongo
import pymongo.errors
from pymongo import MongoClient
from pymongo.errors import ConnectionFailure
except:
print("missing modules")
try:
mongo_client = MongoClient(host="xyz.com", port=27017)
Db = mongo_client['abcd']
coll = Db['shopper_journey_sitedata']
except ConnectionFailure:
print("Connection failed")
date=[]
# Thirdpartyid=[]
input_files = os.listdir(INPUT_FILE_PATH)
# looping through input files
for input_file in input_files:
x = input_file.split(".")
date.append(x[0][5:13])
tp_site_id = x[1]
# print(tp_site_id)
cur = coll.find({"third_party_site_id":tp_site_id})
for doc in cur:
print(doc)
Now i want to parameterize the find() method for every id, so that on each iteration i should get st_site_id ?
above code i tried but ist giving error as "Datas:name error"
You can do one thing
coll.find({"third_party_site_id": { $in :
[318,2624,2621,2622,102,078]}})
If Tid is an array, then you could replace 318 in your query to Tid[I]
I need to import different ipynb files, so I tried this:
https://jupyter-notebook.readthedocs.io/en/stable/examples/Notebook/Importing%20Notebooks.html
But I get no module named 'mynotebook' found. (I even tried it with other notebooks names, which definitely exist, but still not working)
Do you have any ideas about what I could do?
import io, os, sys, types
from IPython import get_ipython
from nbformat import read
from IPython.core.interactiveshell import InteractiveShell
def find_notebook(fullname, path=None):
name = fullname.rsplit('.', 1)[-1]
if not path:
path = ['']
for d in path:
nb_path = os.path.join(d, name + ".ipynb")
if os.path.isfile(nb_path):
return nb_path
# let import Notebook_Name find "Notebook Name.ipynb"
nb_path = nb_path.replace("_", " ")
if os.path.isfile(nb_path):
return nb_path
class NotebookLoader(object):
def __init__(self, path=None):
self.shell = InteractiveShell.instance()
self.path = path
def load_module(self, fullname):
"""import a notebook as a module"""
path = find_notebook(fullname, self.path)
print ("importing Jupyter notebook from %s" % path)
# load the notebook object
with io.open(path, 'r', encoding='utf-8') as f:
nb = read(f, 4)
# create the module and add it to sys.modules
# if name in sys.modules:
# return sys.modules[name]
mod = types.ModuleType(fullname)
mod.__file__ = path
mod.__loader__ = self
mod.__dict__['get_ipython'] = get_ipython
sys.modules[fullname] = mod
# extra work to ensure that magics that would affect the user_ns
# actually affect the notebook module's ns
save_user_ns = self.shell.user_ns
self.shell.user_ns = mod.__dict__
try:
for cell in nb.cells:
if cell.cell_type == 'code':
# transform the input to executable Python
code = self.shell.input_transformer_manager.transform_cell(cell.source)
# run the code in themodule
exec(code, mod.__dict__)
finally:
self.shell.user_ns = save_user_ns
return mod
class NotebookFinder(object):
def __init__(self):
self.loaders = {}
def find_module(self, fullname, path=None):
nb_path = find_notebook(fullname, path)
if not nb_path:
return
key = path
if path:
# lists aren't hashable
key = os.path.sep.join(path)
if key not in self.loaders:
self.loaders[key] = NotebookLoader(path)
return self.loaders[key]
sys.meta_path.append(NotebookFinder())
import mynotebook
I just want to import the code of another jupyter file
WOW, i also face this problem. I create a new env and after open jupyter, it can't find nbformat in my new installed env, so just:
pip install nbformat
Good Morning.
I'm trying to setup a DAG too
Watch/sense for a file to hit a network folder
Process the file
Archive the file
Using the tutorials online and stackoverflow I have been able to come up with the following DAG and Operator that successfully achieves the objectives, however I would like the DAG to be rescheduled or rerun on completion so it starts watching/sensing for another file.
I attempted to set a variable max_active_runs:1 and then a schedule_interval: timedelta(seconds=5) this yes reschedules the DAG but starts queuing task and locks the file.
Any ideas welcome on how I could rerun the DAG after the archive_task?
Thanks
DAG CODE
from airflow import DAG
from airflow.operators import PythonOperator, OmegaFileSensor, ArchiveFileOperator
from datetime import datetime, timedelta
from airflow.models import Variable
default_args = {
'owner': 'glsam',
'depends_on_past': False,
'start_date': datetime.now(),
'provide_context': True,
'retries': 100,
'retry_delay': timedelta(seconds=30),
'max_active_runs': 1,
'schedule_interval': timedelta(seconds=5),
}
dag = DAG('test_sensing_for_a_file', default_args=default_args)
filepath = Variable.get("soucePath_Test")
filepattern = Variable.get("filePattern_Test")
archivepath = Variable.get("archivePath_Test")
sensor_task = OmegaFileSensor(
task_id='file_sensor_task',
filepath=filepath,
filepattern=filepattern,
poke_interval=3,
dag=dag)
def process_file(**context):
file_to_process = context['task_instance'].xcom_pull(
key='file_name', task_ids='file_sensor_task')
file = open(filepath + file_to_process, 'w')
file.write('This is a test\n')
file.write('of processing the file')
file.close()
proccess_task = PythonOperator(
task_id='process_the_file',
python_callable=process_file,
provide_context=True,
dag=dag
)
archive_task = ArchiveFileOperator(
task_id='archive_file',
filepath=filepath,
archivepath=archivepath,
dag=dag)
sensor_task >> proccess_task >> archive_task
FILE SENSOR OPERATOR
import os
import re
from datetime import datetime
from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
from airflow.operators.sensors import BaseSensorOperator
class ArchiveFileOperator(BaseOperator):
#apply_defaults
def __init__(self, filepath, archivepath, *args, **kwargs):
super(ArchiveFileOperator, self).__init__(*args, **kwargs)
self.filepath = filepath
self.archivepath = archivepath
def execute(self, context):
file_name = context['task_instance'].xcom_pull(
'file_sensor_task', key='file_name')
os.rename(self.filepath + file_name, self.archivepath + file_name)
class OmegaFileSensor(BaseSensorOperator):
#apply_defaults
def __init__(self, filepath, filepattern, *args, **kwargs):
super(OmegaFileSensor, self).__init__(*args, **kwargs)
self.filepath = filepath
self.filepattern = filepattern
def poke(self, context):
full_path = self.filepath
file_pattern = re.compile(self.filepattern)
directory = os.listdir(full_path)
for files in directory:
if re.match(file_pattern, files):
context['task_instance'].xcom_push('file_name', files)
return True
return False
class OmegaPlugin(AirflowPlugin):
name = "omega_plugin"
operators = [OmegaFileSensor, ArchiveFileOperator]
Dmitris method worked perfectly.
I also found in my reading setting schedule_interval=None and then using the TriggerDagRunOperator worked equally as well
trigger = TriggerDagRunOperator(
task_id='trigger_dag_RBCPV99_rerun',
trigger_dag_id="RBCPV99_v2",
dag=dag)
sensor_task >> proccess_task >> archive_task >> trigger
Set schedule_interval=None and use airflow trigger_dag command from BashOperator to launch next execution at the completion of the previous one.
trigger_next = BashOperator(task_id="trigger_next",
bash_command="airflow trigger_dag 'your_dag_id'", dag=dag)
sensor_task >> proccess_task >> archive_task >> trigger_next
You can start your first run manually with the same airflow trigger_dag command and then trigger_next task will automatically trigger the next one. We use this in production for many months now and and it runs perfectly.
I'm new to web app and I want to check when there's a new version of dota map, I'll check links in getdota.com.
How can I do this and which language, I want it checks every time you start warcraft, and auto download new map to specific folder.
My question is : Can you give a link to a specific article about web automation or something like that.
Thanks first :)
Below is an example in Python.
It parses getdota.com page, reads parameters for POST request for downloading a map, gets the file and saves it in configured directory (by default current directory).
#!/usr/bin/env python
import urllib
import urllib2
import sgmllib
from pprint import pprint
import os.path
import sys
url = 'http://www.getdota.com/'
download_url = 'http://www.getdota.com/app/getmap/'
chunk = 10000
directory = '' #directory where file should be saved, if empty uses current dir
class DotaParser(sgmllib.SGMLParser):
def parse(self, s):
self.feed(s)
self.close()
def __init__(self, verbose=0):
sgmllib.SGMLParser.__init__(self, verbose)
self.URL = ''
self.post_args = {}
def getArgs(self):
return self.post_args
def start_input(self, attributes):
d = dict(attributes)
if d.get('id', None) == None:
return
if d['id'] in ["input_mirror2", "input_file_name2", "input_map_id2", "input_language2", "input_language_id2"]:
self.post_args[d['name']] = d['value']
if __name__ == '__main__':
dotap = DotaParser()
data = urllib2.urlopen(urllib2.Request('http://www.getdota.com/')).read()
dotap.parse(data)
data = urllib.urlencode(dotap.getArgs())
request = urllib2.Request(download_url, data)
response = urllib2.urlopen(request)
page = response.read()
#download file
fname = directory + page.split('/')[-1]
if os.path.isfile(fname):
print "No newer file available"
sys.exit(0)
f = open(fname, 'w')
print "New file available. Saving in: %s" % fname
webFile = urllib.urlopen(page)
c = webFile.read(chunk)
while(c):
f.write(c)
c = webFile.read(chunk)
f.close()
webFile.close()