I need to generate python grpc code from protobuf files and include them as a packaged module for my project. This is my setup.py
from setuptools import setup, find_namespace_packages
import os
import sys
def generate_grpc():
from grpc_tools import protoc
import pkg_resources
inclusion_root = os.path.abspath("path-to-proto-files")
output_root = os.path.abspath("path-output")
proto_files = []
for root, _, files in os.walk(inclusion_root):
for filename in files:
if filename.endswith('.proto'):
proto_files.append(os.path.abspath(os.path.join(root,
filename)))
well_known_protos_include = pkg_resources.resource_filename(
'grpc_tools', '_proto')
for proto_file in proto_files:
command = [
'grpc_tools.protoc',
'--proto_path={}'.format(inclusion_root),
'--proto_path={}'.format(well_known_protos_include),
'--python_out={}'.format(output_root),
'--grpc_python_out={}'.format(output_root),
] + [proto_file]
if protoc.main(command) != 0:
sys.stderr.write('warning: {} failed'.format(command))
class BuildPyCommand(build_py):
def run(self):
generate_grpc()
build_py.run(self)
setup(cmdclass={
'build_py': BuildPyCommand,
},
setup_requires=[
'grpcio-tools',
],
package_dir={'': 'src'},
packages=find_namespace_packages(where='src')
)
I run this and see that the generated files are not copied to the build directory and as a result are not available when packaged.
Related
I am creating a bazel python project which depends on apache_beam sdk. In the WORKSPACE file of my project, I have downloaded apache_beam zip file via http_archive.
Whenever I am trying to access apache_beam.testing sub-package in my project, I am getting an error
ModuleNotFoundError: No module named 'apache_beam.testing'
import apache_beam is working fine though.
WORKSPACE:
http_archive(
name = "apache_beam",
urls = ["https://github.com/apache/beam/archive/master.zip"],
strip_prefix = 'beam-master/sdks/python',
build_file = str(Label("//third_party:apache_beam.BUILD")),
)
apache_beam.BUILD:
py_library(
name = "apache_beam",
srcs = glob(["apache_beam/**/*.py"]),
visibility = ["//visibility:public"],
)
project's BUILD file:
py_test(
name = "project",
srcs = ["project.py"],
deps = [
"#apache_beam//:apache_beam",
],
project.py file:
from apache_beam.testing import util
In apache_beam.BUILD, you need to set the imports attribute:
py_library(
name = "apache_beam",
srcs = glob(["apache_beam/**/*.py"]),
visibility = ["//visibility:public"],
imports = "apache_beam",
)
I need to import different ipynb files, so I tried this:
https://jupyter-notebook.readthedocs.io/en/stable/examples/Notebook/Importing%20Notebooks.html
But I get no module named 'mynotebook' found. (I even tried it with other notebooks names, which definitely exist, but still not working)
Do you have any ideas about what I could do?
import io, os, sys, types
from IPython import get_ipython
from nbformat import read
from IPython.core.interactiveshell import InteractiveShell
def find_notebook(fullname, path=None):
name = fullname.rsplit('.', 1)[-1]
if not path:
path = ['']
for d in path:
nb_path = os.path.join(d, name + ".ipynb")
if os.path.isfile(nb_path):
return nb_path
# let import Notebook_Name find "Notebook Name.ipynb"
nb_path = nb_path.replace("_", " ")
if os.path.isfile(nb_path):
return nb_path
class NotebookLoader(object):
def __init__(self, path=None):
self.shell = InteractiveShell.instance()
self.path = path
def load_module(self, fullname):
"""import a notebook as a module"""
path = find_notebook(fullname, self.path)
print ("importing Jupyter notebook from %s" % path)
# load the notebook object
with io.open(path, 'r', encoding='utf-8') as f:
nb = read(f, 4)
# create the module and add it to sys.modules
# if name in sys.modules:
# return sys.modules[name]
mod = types.ModuleType(fullname)
mod.__file__ = path
mod.__loader__ = self
mod.__dict__['get_ipython'] = get_ipython
sys.modules[fullname] = mod
# extra work to ensure that magics that would affect the user_ns
# actually affect the notebook module's ns
save_user_ns = self.shell.user_ns
self.shell.user_ns = mod.__dict__
try:
for cell in nb.cells:
if cell.cell_type == 'code':
# transform the input to executable Python
code = self.shell.input_transformer_manager.transform_cell(cell.source)
# run the code in themodule
exec(code, mod.__dict__)
finally:
self.shell.user_ns = save_user_ns
return mod
class NotebookFinder(object):
def __init__(self):
self.loaders = {}
def find_module(self, fullname, path=None):
nb_path = find_notebook(fullname, path)
if not nb_path:
return
key = path
if path:
# lists aren't hashable
key = os.path.sep.join(path)
if key not in self.loaders:
self.loaders[key] = NotebookLoader(path)
return self.loaders[key]
sys.meta_path.append(NotebookFinder())
import mynotebook
I just want to import the code of another jupyter file
WOW, i also face this problem. I create a new env and after open jupyter, it can't find nbformat in my new installed env, so just:
pip install nbformat
I have a pytest testing project running selenium tests that has a structure like:
ProjRoot
|
|_Pytest.ini
|_____________TestFolderA
| |
| |_test_folderA_tests1.py
| |_test_folderA_tests2.py
|
|____________TestFolderB
| |
| |_test_folderB_test1.py
| |_test_folderA_tests2.py
|
|
|___________TestHelperModules
| |
| |_VariousTestHelperModules
|
|____________DriversAndTools
|___(contains chromedriver.exe, firefox profile folder etc)
I have a confTest.py file which I currently run in the ProjRoot, which I use as a setup and tear down for establishing the browser session for each test that is run. It runs each test twice. Once for Chrome and once for Firefox. In my tests I just utilise the resulting driver fixture. The conftest file is as below:
#conftest.py
import pytest
import os
import rootdir_ref
from selenium.webdriver.common.keys import Keys
import time
from webdriverwrapper.pytest import *
from webdriverwrapper import Chrome
from webdriverwrapper import DesiredCapabilities
from webdriverwrapper import Firefox
from webdriverwrapper import FirefoxProfile
#when running tests from command line we should be able to pass --url=www..... for a different website, check what order these definitions need to be in
def pytest_addoption(parser):
parser.addoption('--url', default='https://test1.testsite.com.au')
#pytest.fixture(scope='function')
def url(request):
return request.config.option.url
browsers = {
'firefox': Firefox,
'chrome': Chrome,
}
#pytest.fixture(scope='function',
params=browsers.keys())
def browser(request):
if request.param == 'firefox':
firefox_capabilities = DesiredCapabilities.FIREFOX
firefox_capabilities['marionette'] = True
firefox_capabilities['handleAlerts'] = True
theRootDir = os.path.dirname(rootdir_ref.__file__)
ffProfilePath = os.path.join(theRootDir, 'DriversAndTools', 'FirefoxSeleniumProfile')
geckoDriverPath = os.path.join(theRootDir, 'DriversAndTools', 'geckodriver.exe')
profile = FirefoxProfile(profile_directory=ffProfilePath)
print (ffProfilePath)
print (geckoDriverPath)
b = browsers[request.param](firefox_profile=profile, capabilities=firefox_capabilities, executable_path=geckoDriverPath)
elif request.param == 'chrome':
desired_cap = DesiredCapabilities.CHROME
desired_cap['chromeOptions'] = {}
desired_cap['chromeOptions']['args'] = ['--disable-plugins', '--disable-extensions']
theRootDir = os.path.dirname(rootdir_ref.__file__)
chromeDriverPath = os.path.join(theRootDir, 'DriversAndTools', 'chromedriver.exe')
b = browsers[request.param](chromeDriverPath)
else:
b = browsers[request.param]()
request.addfinalizer(lambda *args: b.quit())
return b
#pytest.fixture(scope='function')
def driver(browser, url):
driver = browser
driver.maximize_window()
driver.get(url)
return driver
What I’d like to do is have a conftest file in each Test Folder instead of the ProjRoot. But if I take this existing conftest file and put it in each test folder and then run pytest from the project root using
python –m pytest
letting pytest pickup the test directories from pytest.ini (expecting the test folders to run with their respectively contained conftest files) I have issues with the parser.addoption --url already having been added. The end of the error message is:
ClientScripts\conftest.py:19: in pytest_addoption
parser.addoption('--url', default='https://test1.coreplus.com.au/coreplus01')
..\..\..\VirtEnv\VirtEnv\lib\site-packages\_pytest\config.py:521: in addoption
self._anonymous.addoption(*opts, **attrs)
..\..\..\VirtEnv\VirtEnv\lib\site-packages\_pytest\config.py:746: in addoption
raise ValueError("option names %s already added" % conflict)
E ValueError: option names {'--url'} already added
The purpose of the --url addoption is so I can override the defaults in the conftest file at commandline if I want to point them all to a different url at the same time, but otherwise let them default to running to different url's as specified in their conftest files.
I had a similar issue.
Error was gone after removing all cached files and venv.
I'm trying to do some basic module setups on my server using Python. Its a bit difficult as I have no access to the internet.
This is my code
import sys
import os
from subprocess import CalledProcessError, STDOUT, check_output
def run_in_path(command, dir_path, env_var=''):
env_var = os.environ["PATH"] = os.environ["PATH"] + env_var
print(env_var)
try:
p = check_output(command, cwd=dir_path, stderr=STDOUT)
except CalledProcessError as e:
sys.stderr.write(e.output.decode("utf-8"))
sys.stderr.flush()
return e.returncode
else:
return 0
def main():
requests_install = run_in_path('python setup.py build',
'D:\installed_software\python modules\kennethreitz-requests-e95e173')
SQL_install = run_in_path('python setup.py install', # install SQL module pypyodbc
'D:\installed_software\python modules\pypyodbc-1.3.3\pypyodbc-1.3.3')
setup_tools = run_in_path('python setup.py install', # install setup tools
'D:\installed_software\python modules\setuptools-17.1.1')
psycopg2 = run_in_path('easy_install psycopg2-2.6.1.win-amd64-py3.3-pg9.4.4-release', # install setup tools
'D:\installed_software\python modules', ';C:\srv_apps\Python33\Scripts\easy_install.exe')
print('setup complete')
if __name__ == "__main__":
sys.exit(main())
now it gets tricky when i start trying to use easy install. It appears my env variables are not being used by my subprocess.check_output call
File "C:\srv_apps\Python33\lib\subprocess.py", line 1110, in _execute_child
raise WindowsError(*e.args)
FileNotFoundError: [WinError 2] The system cannot find the file specified
I don't want to have to upgrade to 3.4 where easy install is installed by default because my other modules are not supported on 3.4. My main challenge is the subprocess.check_call method does not take environment variables as an input and im wary of trying to use Popen() as I have never really got it to work successfully in the past. Any help would be greatly appreciated.
PATH should contain directories e.g., r'C:\Python33\Scripts', not files such as: r'C:\Python33\Scripts\easy_install.exe'
Don't hardcode utf-8 for an arbitrary command, you could enable text mode using universal_newlines parameter (not tested):
#!/usr/bin/env python3
import locale
import sys
from subprocess import CalledProcessError, STDOUT, check_output
def run(command, *, cwd=None, env=None):
try:
ignored = check_output(command, cwd=cwd, env=env,
stderr=STDOUT,
universal_newlines=True)
except CalledProcessError as e:
sys.stderr.write(e.output)
sys.stderr.flush()
return e.returncode
else:
return 0
Example:
import os
path_var = os.pathsep.join(os.environ.get('PATH', os.defpath), some_dir)
env = dict(os.environ, PATH=path_var)
run("some_command", cwd=some_path, env=env)
run("another_command", cwd=another_path, env=env)
I have 4 .py files. Below is a list of files what is required to run the programme. Any of them missing will fail the programme to run.
How my code works:
) GUIss.py imports demonstrator.py
) Demonstrator.py imports filereader.py and process.py
) To run the programm I just need to click GUIss.py.
My cx-freeze code below:
from cx_Freeze import setup,Executable
import os
includefiles = ['filereader.py','demonstrator.py','logo.gif','thebrighterchoice.gif']
#bin_includes= ['process.py','demonstrator.py','filereader.py'] ..... 'bin_includes':bin_includesincludes = ['process']
includes = ['process','tkinter']
excludes = ['tkinter']
packages = ['os','xlrd']
setup(
name = "Process",
version = "0.1",
description = "description",
author = "Raitis Kupce",
options = {'build_exe' :{'excludes': excludes,'includes':includes,'packages':packages,'include_files':includefiles}},
executables = [Executable("GUIss.py")]
)
When I run compiled file I get an error message:
I then tried to write in setup.py (cx-freeze file)
excludes = ['tkinter']
Then includes = ['tkinter']
Afterwards packages = ['tkinter']
Despite numerous attempt, no luck, same message all the time.
P.S
My python source code can be downloaded from https://github.com/Dragnets/Administration
I did studied hours from here and here and modifying endless times but no luck.