pytest - mockup a complex module import - pytest

I have found several posts on how to "hide" a package and simulate an ImportError with pytest, however, I haven't succeeded in my case and I am looking for some help:
Test for import of optional dependencies in __init__.py with pytest: Python 3.5 /3.6 differs in behaviour
Test behavior of code if optional module is not installed
and related
Here is the content of an __about__.py file that I want to test with pytest.
"""Get the metadata from the package or from setup.py."""
try:
import importlib
metadata = importlib.metadata
except ImportError:
import importlib_metadata as metadata
try:
data = metadata.metadata("mypackage")
__version__ = data["Version"]
__author__ = data["Author"]
__name__ = data["Name"]
except metadata.PackageNotFoundError:
# The repo of the package is accessible to python to get at least the version
import re
from pathlib import Path
try:
from nested_grid_plotter import __file__ as loc
with open(Path(loc).parent.joinpath("../setup.py"), "r") as f:
data = f.read()
except FileNotFoundError:
data = ""
def version_parser(v):
"""Parse the version from the setup file."""
version_pattern = (
r"""(version\s*=\s*)["|'](\d+(=?\.(\d+(=?\.(\d+)*)*)*)*)["|']"""
)
regex_matcher = re.compile(version_pattern).search(v)
if regex_matcher is None:
return "unknwon"
return regex_matcher.group(2)
try:
__version__ = version_parser(data)
except Exception:
__version__ = "unknown"
__author__ = "unknown"
__name__ = "unknown"
Here is the __init__.py at the root of the package:
from .__about__ import __version__, __name__, __author__
And here is the tests that I have come up with until now. However, I am not able to hide importlib.
"""Test the file __about__.py."""
import pytest
import sys
class PackageDiscarder:
def __init__(self):
self.pkgnames = []
def find_spec(self, fullname, path, target=None):
if fullname in self.pkgnames:
raise ImportError()
#pytest.fixture
def no_requests():
sys.modules.pop("importlib", None)
d = PackageDiscarder()
d.pkgnames.append("importlib")
sys.meta_path.insert(0, d)
yield
sys.meta_path.remove(d)
#pytest.fixture(autouse=True)
def cleanup_imports():
yield
sys.modules.pop("mypackage", None)
def test_requests_available():
import mypackage
assert mypackage.__version__ != "unknwon"
#pytest.mark.usefixtures("no_requests")
def test_requests_missing():
import mypackage
assert mypackage.__version__ != "unknwon"
Here is the coverage report:
Name Stmts Miss Cover Missing
----------------------------------------------------------------
mypackage/__about__.py 31 10 68% 5-6, 10-12, 23-24, 33, 38-39
----------------------------------------------------------------
TOTAL 31 10 68%

Related

Is there way to show test author name in pytest-html reports for each tests?

Is there way to show test author name in pytest-html reports for each tests ?
Yes. You can do that. Put __author__ attribute in each test module.
# test_add.py
__author__ = 'Vivek R'
def add(x, y):
return x + y
def test_add():
assert add(5, 4) == 9
And call some fixtures in conftest.py file.
# conftest.py
from py.xml import html
import pytest
def pytest_html_results_table_header(cells):
cells.append(html.th("Author"))
def pytest_html_results_table_row(report, cells):
cells.append(html.td(report.author))
#pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
outcome = yield
report = outcome.get_result()
report.author = item.module.__author__

Copy contents of tmp_path -> tmp_path_factory for use in the next test

Test_1 writes out a file tree to the standard tmp_path fixture.
Test_2 requires this file tree as the input to its own test.
I am using shutil.copytree() however I am getting an error:
import pytest
import shutil
import os
def test_1(tmp_path, tmp_path_factory):
p = tmp_path / "me"
shutil.copytree(p, tmp_path_factory, dirs_exist_ok=True)
assert True
def test_2(tmp_path_factory):
l = list(tmp_path_factory.iterdir())
print(l)
assert False
Error:
p = TempPathFactory(_given_basetemp=None, _trace=<pluggy._tracing.TagTracerSub object at 0x000002161F3F4BE0>, _basetemp=WindowsPath('C:/Users/David/AppData/Local/Temp/pytest-of-David/pytest-1125'))
def split(p):
"""Split a pathname.
Return tuple (head, tail) where tail is everything after the final slash.
Either part may be empty."""
> p = os.fspath(p)
E TypeError: expected str, bytes or os.PathLike object, not TempPathFactory
Any ideas? Many thanks :)
Thanks to #hoefling, this works
import pytest
import shutil
from directory_tree import display_tree
#pytest.fixture(scope="session")
def cache_dir(tmp_path_factory):
return tmp_path_factory.mktemp("cache")#, numbered=False)
def test_1(tmp_path, cache_dir):
p = tmp_path / "me.txt"
p.touch()
shutil.copytree(p.parent, cache_dir, dirs_exist_ok=True)
assert True
def test_2(cache_dir):
display_tree(cache_dir)
assert False

import class with pandas from file

I have two files:
main.py
from personal_function.Convert import cls_Convert
df_key = 'PERFIL'
a = cls_Convert(df_data_2)
a.convert_cat_to_num(df_key)
df_data_2 = a.df
personal_function/Convert.py
import pandas as pd
class cls_Convert:
def __init__(self,df):
self.df = df
# Mudar variavel categorica para numerica
def convert_cat_to_num(self,df_key):
self.df[df_key] = pd.factorize(self.df[df_key],sort=True)[0] + 1
return self.df
# Mudar variavel numerica para categorica
def convert_num_to_cat(self,df_key,cat_bin,cat_label):
self.df[df_key].replace(to_replace = cat_bin, value =cat_label, inplace=True)
return self.df
however I get this error
ImportError: cannot import name 'cls_Convert' from 'personal_function.Convert'
For a class or function to be visible outside of a package, it must be imported in the package's __init__.py file which is run when the package is imported from somewhere. All the variables, imports, method, and classes defined in that __init__.py are then made visible to the package that was importing them. Take the below example:
example/example.py
def visible():
pass
def not_visible():
pass
example/init.py
from .ex import visible
main.py
from example import visible
from example import not_visible # results in an error since it was not imported
# in the `example` package's `__init__.py` file.
To make your Convert class visible to the external main.py file, create the __init__.py for the package.
You can read more about python submoduling here
I need to use
from os import getcwd
from sys import path
cwd = getcwd()
path.append(cwd)
than use the whole absolute path

Error in running Apache Beam Python SplittableDoFn

Error encountered while trying pubsub io > splittable dofn
RuntimeError: Transform node
AppliedPTransform(ParDo(TestDoFn)/ProcessKeyedElements/GroupByKey/GroupByKey,
_GroupByKeyOnly) was not replaced as expected.
Can someone help me with reviewing the code for anything I might be doing incorrectly in there
Code:
"""
python examples/test_restriction_unbounded.py --project mk2 --topic projects/mk2/topics/testing
"""
# pytype: skip-file
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import csv
import logging
import sys
import time
from datetime import datetime
import apache_beam as beam
from apache_beam.options.pipeline_options import GoogleCloudOptions
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.options.pipeline_options import StandardOptions
from apache_beam.io.restriction_trackers import OffsetRestrictionTracker, OffsetRange
from apache_beam.transforms.core import RestrictionProvider
class TestProvider(RestrictionProvider):
def initial_restriction(self, element):
return OffsetRange(0, 1)
def create_tracker(self, restriction):
return OffsetRestrictionTracker(restriction)
def restriction_size(self, element, restriction):
return restriction.size()
class TestDoFn(beam.DoFn):
def process(
self,
element,
restriction_tracker=beam.DoFn.RestrictionParam(
TestProvider())):
import pdb; pdb.set_trace()
cur = restriction_tracker.current_restriction().start
while restriction_tracker.try_claim(cur):
return element
def run(argv=None, save_main_session=True):
parser = argparse.ArgumentParser()
parser.add_argument('--topic', type=str, help='Pub/Sub topic to read from')
args, pipeline_args = parser.parse_known_args(argv)
options = PipelineOptions(pipeline_args)
options.view_as(StandardOptions).streaming = True
with beam.Pipeline(options=options) as p:
# data = ['abc', 'defghijklmno', 'pqrstuv', 'wxyz']
# actual = (p | beam.Create(data) | beam.ParDo(ExpandingStringsDoFn()))
scores = p | beam.io.ReadFromPubSub(topic=args.topic) | beam.ParDo(TestDoFn())
if __name__ == '__main__':
logging.getLogger().setLevel(logging.INFO)
run()
You are ingesting data from pub/sub by steaming. Then you have to create batches by window before apply this kind of transforms: (ParDo(TestDoFn)/ProcessKeyedElements/GroupByKey/GroupByKey, _GroupByKeyOnly)
Pub/Sub with window example: https://cloud.google.com/pubsub/docs/pubsub-dataflow
Try to do like this:
class GroupWindowsIntoBatches(beam.PTransform):
"""A composite transform that groups Pub/Sub messages
"""
def __init__(self, window_size):
# Convert minutes into seconds.
self.window_size = int(window_size * 60)
def expand(self, pcoll):
return (
pcoll
# Assigns window info to each Pub/Sub message based on its
# publish timestamp.
| "Window into Fixed Intervals"
>> beam.WindowInto(window.FixedWindows(self.window_size))
)
def run(argv=None, save_main_session=True):
parser = argparse.ArgumentParser()
parser.add_argument('--topic', type=str, help='Pub/Sub topic to read from')
args, pipeline_args = parser.parse_known_args(argv)
options = PipelineOptions(pipeline_args)
options.view_as(StandardOptions).streaming = True
window_size = 1.0
with beam.Pipeline(options=options) as p:
scores = (p
| beam.io.ReadFromPubSub(topic=args.topic)
| "WindowInto" >> GroupWindowsIntoBatches(window_size)
| beam.ParDo(TestDoFn())
)
I had the same error. Removing the streaming option solved the problem for me.

pytest with classes python

I wrote the following code :
publisher.py:
import six
from google.api_core.exceptions import AlreadyExists
from google.cloud.pubsub import types
class publisher(object):
"""Publisher Object which has the following attributes
Attributes:
pubsub: publisher client
project_name: Name of project
topic_name: Name of topic
"""
def __init__(self, pubsub, project_name, topic_name, batch_settings=(), *args, **kwargs):
self.pubsub = pubsub
self.project_name = project_name
self.topic_name = topic_name
self.batch_settings = types.BatchSettings(
*batch_settings) # Batch setting Pub/Sub accepts a maximum of 1,000 messages in a batch,
# and the size of a batch can not exceed 10 megabytes
def _full_project_name(self):
"""Returns Fully Qualified Name of project"""
return self.pubsub.project_path(self.project_name)
and I wrote 3 test unfortunately the third one has been failing.
Below is the code I wrote for tests:
test_publisher.py:
from google.cloud import pubsub
import pytest
from publisher import publisher
PROJECT = 'ProjectTest'
TOPIC_NAME = 'TopicTest'
#pytest.fixture
def pubsub():
yield pubsub.PublisherClient()
def test_init_value():
sample_publisher=publisher(pubsub,PROJECT,TOPIC_NAME,())
assert sample_publisher.project_name == 'ProjectTest'
assert sample_publisher.topic_name == 'TopicTest'
assert sample_publisher.pubsub == pubsub
assert sample_publisher.batch_settings.max_messages == 1000
assert sample_publisher.batch_settings.max_bytes == 10 * (2 ** 20)
assert sample_publisher.batch_settings.max_latency == 0.05
def test_init_with_no_values():
with pytest.raises(Exception) as e_info:
sample_bad_init = publisher()
def test_full_project_name ():
sample_publisher = publisher(pubsub, PROJECT, TOPIC_NAME, ())
assert sample_publisher._full_project_name() == 'projects/ProjectTest'
I am currently getting the following error, which I can't understand, unfortunately:
line 26, in _full_project_name
return self.pubsub.project_path(self.project_name)
AttributeError: 'function' object has no attribute 'project_path'
Any help with this, please.
Thanks a lot
The name of fixture should be changed.
#pytest.fixture
def google_pubsub():
yield pubsub.PublisherClient()
You should add google_pubsub as argument to test test_full_project_name(google_pubsub) and test_init_value(google_pubsub).
In Test test_init_value you use module pubsub imported from from google.cloud import pubsub, what is wrong.
Test test_init_value passes because you comparing module(pubsub) in line
assert sample_publisher.pubsub == google_pubsub