Unable to use pytest parameterize functionality to update the YAML config - pytest

I am using ruamel.yaml module to update the YAML using python.
I wanted to use pytest parameterize functionality to update the YAML config.
Here is the test.yaml file:
TestConfig:
hostname: 10.2.4.6
Organisation:
Employee:
Name: Smriti
Skilss: Python
OrganizationName: ABC
UserId: smriti_test#gmail.com
Here is the conftest file so that we need to not to explicitly import the fixture functionality:
from ruamel.yaml import YAML
import pytest
#pytest.fixture(scope='function')
def yaml_loader(request):
yaml = YAML()
file_path = 'test.yaml'
with open(file_path) as fp:
data = yaml.load(fp)
test = data['TestConfig']
x = request.param
print(x)
with open(file_path, "w") as file:
yaml.dump(data, file)
print(data)
Here is the implementation of sample testfile for running the test and using the fixture from conftest and update the config in YAML and perform the test.
import pytest
class TestYAML:
""" TestYAML """
#pytest.mark.parametrize("yaml_loader",[("test['Organisation']['Employee']\
['Name']='Arushi'")],indirect=True)
#pytest.mark.usefixtures("yaml_loader")
def test_update_yamlconfig(self):
pass
In the result I am seeing the x is printing the updated Value of Name to Arushi but in the YAML file the config is not updating.

After couple of tries i am able to find the solution to this question.
So Here is the updated code for reference:-
For test file i have taken two different param and i will assign that param individually when i call the fixtures and this has worked
test_yaml.py
import pytest
class TestYAML:
""" TestYAML """
#pytest.mark.parametrize("yaml_loader",
[("hostname","10.5.6.8")],indirect=True)
#pytest.mark.usefixtures("yaml_loader")
def test_update_yamlconfig(self):
pass
Here is the updated conftest where i have defined the fixture:-
from ruamel.yaml import YAML
import pytest
#pytest.fixture(scope='function')
def yaml_loader(request):
yaml = YAML()
file_path = 'test.yaml'
file_path1 = 'my.yaml'
with open(file_path) as fp:
data = yaml.load(fp)
test = data['TestConfig']
test[request.param[0]] = request.param[1]
with open(file_path1, "w") as file:
yaml.dump(data, file)
print(data)
And you will get the output as :-
platform linux -- Python 3.8.0, pytest-5.4.3, py-1.9.0, pluggy-0.13.1 --
/mnt/c/Users/smaheshw/PycharmProjects/YAML/venv/bin/python3.8
cachedir: .pytest_cache
rootdir: /mnt/c/Users/smaheshw/PycharmProjects/YAML
collected 1 item
test_yaml.py::TestYAML::test_update_yamlconfig[yaml_loader0] Request parameters
hostname
ordereddict([('TestConfig', ordereddict([('hostname', '10.5.6.8'), ('Organisation',
ordereddict([('Employee', ordereddict([('Name', 'Smriti'), ('Skilss', 'Python'),
('OrganizationName', 'A
BC'), ('UserId', 'smriti_test#gmail.com')]))]))]))])
PASSED

If you are stuck with pytest not behaving as you want, you should take a step back and make sure
your yaml_laoder() function does what you expect:
import sys
from pathlib import Path
from ruamel.yaml import YAML
yaml_str = """\
TestConfig:
hostname: 10.2.4.6
Organisation:
Employee:
Name: Smriti
Skilss: Python
OrganizationName: ABC
UserId: smriti_test#gmail.com
"""
Path('test.yaml').write_text(yaml_str) # write out the file so that it is fresh every time
class R: pass
r = R()
r.param = "XXXXXX"
def yaml_loader(request):
yaml = YAML()
file_path = 'test.yaml'
with open(file_path) as fp:
data = yaml.load(fp)
test = data['TestConfig']
x = request.param
print(x)
with open(file_path, "w") as file:
yaml.dump(data, file)
print(data)
yaml_loader(r)
print('\n######### YAML #########\n')
print(Path('test.yaml').read_text())
which gives:
XXXXXX
ordereddict([('TestConfig', ordereddict([('hostname', '10.2.4.6'), ('Organisation', ordereddict([('Employee', ordereddict([('Name', 'Smriti'), ('Skilss', 'Python'), ('OrganizationName', 'ABC'), ('UserId', 'smriti_test#gmail.com')]))]))]))])
######### YAML #########
TestConfig:
hostname: 10.2.4.6
Organisation:
Employee:
Name: Smriti
Skilss: Python
OrganizationName: ABC
UserId: smriti_test#gmail.com
As you can clearly sea from the YAML content, it never gets updated with XXXXXX. This is because you don't assign to data
before writing out the test.yaml file.
Fix that first and then inject the code into your test.

Related

pytest - mockup a complex module import

I have found several posts on how to "hide" a package and simulate an ImportError with pytest, however, I haven't succeeded in my case and I am looking for some help:
Test for import of optional dependencies in __init__.py with pytest: Python 3.5 /3.6 differs in behaviour
Test behavior of code if optional module is not installed
and related
Here is the content of an __about__.py file that I want to test with pytest.
"""Get the metadata from the package or from setup.py."""
try:
import importlib
metadata = importlib.metadata
except ImportError:
import importlib_metadata as metadata
try:
data = metadata.metadata("mypackage")
__version__ = data["Version"]
__author__ = data["Author"]
__name__ = data["Name"]
except metadata.PackageNotFoundError:
# The repo of the package is accessible to python to get at least the version
import re
from pathlib import Path
try:
from nested_grid_plotter import __file__ as loc
with open(Path(loc).parent.joinpath("../setup.py"), "r") as f:
data = f.read()
except FileNotFoundError:
data = ""
def version_parser(v):
"""Parse the version from the setup file."""
version_pattern = (
r"""(version\s*=\s*)["|'](\d+(=?\.(\d+(=?\.(\d+)*)*)*)*)["|']"""
)
regex_matcher = re.compile(version_pattern).search(v)
if regex_matcher is None:
return "unknwon"
return regex_matcher.group(2)
try:
__version__ = version_parser(data)
except Exception:
__version__ = "unknown"
__author__ = "unknown"
__name__ = "unknown"
Here is the __init__.py at the root of the package:
from .__about__ import __version__, __name__, __author__
And here is the tests that I have come up with until now. However, I am not able to hide importlib.
"""Test the file __about__.py."""
import pytest
import sys
class PackageDiscarder:
def __init__(self):
self.pkgnames = []
def find_spec(self, fullname, path, target=None):
if fullname in self.pkgnames:
raise ImportError()
#pytest.fixture
def no_requests():
sys.modules.pop("importlib", None)
d = PackageDiscarder()
d.pkgnames.append("importlib")
sys.meta_path.insert(0, d)
yield
sys.meta_path.remove(d)
#pytest.fixture(autouse=True)
def cleanup_imports():
yield
sys.modules.pop("mypackage", None)
def test_requests_available():
import mypackage
assert mypackage.__version__ != "unknwon"
#pytest.mark.usefixtures("no_requests")
def test_requests_missing():
import mypackage
assert mypackage.__version__ != "unknwon"
Here is the coverage report:
Name Stmts Miss Cover Missing
----------------------------------------------------------------
mypackage/__about__.py 31 10 68% 5-6, 10-12, 23-24, 33, 38-39
----------------------------------------------------------------
TOTAL 31 10 68%

Pytest: How to parametrize input

Is there a way to pass input to pytest.mark.parametrize()?
If I try…
import pytest
from typing import List
#pytest.mark.parametrize("rotation", range(len(input_sequence)))
def test_sequence_rotation(
input_sequence: List[float],
rotation: int,
) -> None:
sequence = input_sequence[rotation:] + input_sequence[:rotation]
print(f"Testing sequence {sequence}")
… I get NameError: name 'input_sequence' is not defined.
For some context, I have input_sequence defined as a pytest command option in conftest.py:
import pytest
from typing import List
from _pytest.config.argparsing import Parser
from _pytest.fixtures import SubRequest
def pytest_addoption(parser: Parser) -> None:
parser.addoption("--sequence-csv", type=str, required=True)
#pytest.fixture()
def input_sequence(request: SubRequest) -> List[float]:
csv = request.config.getoption("--sequence-csv")
return [float(i) for i in csv.split(",")]
You're referring to a variable input_sequence in your call to #pytest.mark.parametrize, but no such variable is in scope. You would need to define this at the module level (either directly or by importing it from somewhere).
You would also need to remove it from the parameter list of test_sequence_rotation.
E.g., something like:
import pytest
from typing import List
input_sequence = ['a', 'b', 'c', 'd', 'e']
#pytest.mark.parametrize("rotation", range(len(input_sequence)))
def test_sequence_rotation(rotation: int) -> None:
sequence = input_sequence[rotation:] + input_sequence[:rotation]
print(f"Testing sequence {sequence}")
The above code will produce output like this when run via pytest -v:
============================= test session starts ==============================
platform linux -- Python 3.9.4, pytest-6.0.2, py-1.10.0, pluggy-0.13.1 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /home/lars/tmp/python
plugins: cov-2.11.1, flake8-1.0.7, asyncio-0.14.0, xonsh-0.9.26
collecting ... collected 5 items
test_sequence.py::test_sequence_rotation[0] PASSED [ 20%]
test_sequence.py::test_sequence_rotation[1] PASSED [ 40%]
test_sequence.py::test_sequence_rotation[2] PASSED [ 60%]
test_sequence.py::test_sequence_rotation[3] PASSED [ 80%]
test_sequence.py::test_sequence_rotation[4] PASSED [100%]
============================== 5 passed in 0.03s ===============================

import class with pandas from file

I have two files:
main.py
from personal_function.Convert import cls_Convert
df_key = 'PERFIL'
a = cls_Convert(df_data_2)
a.convert_cat_to_num(df_key)
df_data_2 = a.df
personal_function/Convert.py
import pandas as pd
class cls_Convert:
def __init__(self,df):
self.df = df
# Mudar variavel categorica para numerica
def convert_cat_to_num(self,df_key):
self.df[df_key] = pd.factorize(self.df[df_key],sort=True)[0] + 1
return self.df
# Mudar variavel numerica para categorica
def convert_num_to_cat(self,df_key,cat_bin,cat_label):
self.df[df_key].replace(to_replace = cat_bin, value =cat_label, inplace=True)
return self.df
however I get this error
ImportError: cannot import name 'cls_Convert' from 'personal_function.Convert'
For a class or function to be visible outside of a package, it must be imported in the package's __init__.py file which is run when the package is imported from somewhere. All the variables, imports, method, and classes defined in that __init__.py are then made visible to the package that was importing them. Take the below example:
example/example.py
def visible():
pass
def not_visible():
pass
example/init.py
from .ex import visible
main.py
from example import visible
from example import not_visible # results in an error since it was not imported
# in the `example` package's `__init__.py` file.
To make your Convert class visible to the external main.py file, create the __init__.py for the package.
You can read more about python submoduling here
I need to use
from os import getcwd
from sys import path
cwd = getcwd()
path.append(cwd)
than use the whole absolute path

pytest with classes python

I wrote the following code :
publisher.py:
import six
from google.api_core.exceptions import AlreadyExists
from google.cloud.pubsub import types
class publisher(object):
"""Publisher Object which has the following attributes
Attributes:
pubsub: publisher client
project_name: Name of project
topic_name: Name of topic
"""
def __init__(self, pubsub, project_name, topic_name, batch_settings=(), *args, **kwargs):
self.pubsub = pubsub
self.project_name = project_name
self.topic_name = topic_name
self.batch_settings = types.BatchSettings(
*batch_settings) # Batch setting Pub/Sub accepts a maximum of 1,000 messages in a batch,
# and the size of a batch can not exceed 10 megabytes
def _full_project_name(self):
"""Returns Fully Qualified Name of project"""
return self.pubsub.project_path(self.project_name)
and I wrote 3 test unfortunately the third one has been failing.
Below is the code I wrote for tests:
test_publisher.py:
from google.cloud import pubsub
import pytest
from publisher import publisher
PROJECT = 'ProjectTest'
TOPIC_NAME = 'TopicTest'
#pytest.fixture
def pubsub():
yield pubsub.PublisherClient()
def test_init_value():
sample_publisher=publisher(pubsub,PROJECT,TOPIC_NAME,())
assert sample_publisher.project_name == 'ProjectTest'
assert sample_publisher.topic_name == 'TopicTest'
assert sample_publisher.pubsub == pubsub
assert sample_publisher.batch_settings.max_messages == 1000
assert sample_publisher.batch_settings.max_bytes == 10 * (2 ** 20)
assert sample_publisher.batch_settings.max_latency == 0.05
def test_init_with_no_values():
with pytest.raises(Exception) as e_info:
sample_bad_init = publisher()
def test_full_project_name ():
sample_publisher = publisher(pubsub, PROJECT, TOPIC_NAME, ())
assert sample_publisher._full_project_name() == 'projects/ProjectTest'
I am currently getting the following error, which I can't understand, unfortunately:
line 26, in _full_project_name
return self.pubsub.project_path(self.project_name)
AttributeError: 'function' object has no attribute 'project_path'
Any help with this, please.
Thanks a lot
The name of fixture should be changed.
#pytest.fixture
def google_pubsub():
yield pubsub.PublisherClient()
You should add google_pubsub as argument to test test_full_project_name(google_pubsub) and test_init_value(google_pubsub).
In Test test_init_value you use module pubsub imported from from google.cloud import pubsub, what is wrong.
Test test_init_value passes because you comparing module(pubsub) in line
assert sample_publisher.pubsub == google_pubsub

mocking snowflake connection

I have a SnowflakeApi class in python which just works as a wrapper on top of the SnowflakeConnection class. My SnowflakeApi is
import logging
import os
from snowflake.connector import connect
class SnowflakeApi(object):
"""
Wrapper to handle snowflake connection
"""
def __init__(self, account, warehouse, database, user, pwd):
"""
Handles snowflake connection. Connection must be closed once it is no longer needed
:param account:
:param warehouse:
:param database:
"""
self.__acct = self._account_url(account)
self.__wh = warehouse
self.__db = database
self.__connection = None
self.__user = user
self.__pwd = pwd
def __create_connection(self):
try:
# set the proxy here
conn = connect(
account=self.__acct
, user=self.__user
, password=self.__pwd
, warehouse=self.__wh
, database=self.__db
)
return conn
except:
raise Exception(
"Unable to connect to snowflake for user: '{0}', warehouse: '{1}', database: '{2}'".format(
self.__user, self.__wh, self.__db))
def get_connection(self):
"""
Gets a snowflake connection. If the connection has already been initialised it is returned
otherwise a new connection is created
:param credentials_func: method to get database credentials.
:return:
"""
try:
if self.__connection is None:
self.__connection = self.__create_connection()
return self.__connection
except:
raise Exception("Unable to initalise Snowflake connection")
def close_connection(self):
"""
Closes snowflake connection.
:return:
"""
self.__connection.close()
Namespace for SnowflakeApi is connection.snowflake_connection.SnowflakeApi (i.e. i have snowflake_connection.py in a folder called connections)
I want to write unit tests for this class using pytest and unittest.mock. The problem is I want to mock 'connect' so that a MagicMock object is returned and no database call is made. So far I have tried:
monkeypatch.setattr(connections.snowflake_connection,"connect",return_value = "")
Changed my original class to just import snowflake. I then created a mock object and used monkeypatch.setattr(snowflake_connection,"snowflake",my_mock_snowflake). That didn't work either
In short, I have tried a couple of other things but nothing has worked. All I want to do is mock snowflake connection so no actual database call is made.
Here is another way where we are mocking snowflake connector, cursor and fetch_all using python mock and patch.
import mock
import unittest
from datetime import datetime, timedelta
import feed_daily_report
class TestFeedDailyReport(unittest.TestCase):
#mock.patch('snowflake.connector.connect')
def test_compare_partner(self, mock_snowflake_connector):
tod = datetime.now()
delta = timedelta(days=8)
date_8_days_ago = tod - delta
query_result = [('partner_1', date_8_days_ago)]
mock_con = mock_snowflake_connector.return_value
mock_cur = mock_con.cursor.return_value
mock_cur.fetchall.return_value = query_result
result = feed_daily_report.main()
assert result == True
An example using unittest.mock and patching the connection:
from unittest import TestCase
from unittest.mock import patch
from connection.snowflake_connection import SnowflakeApi
class TestSnowFlakeApi(TestCase):
#patch('connection.snowflake_connection.connect')
def test_get_connection(self, mock_connect)
api = SnowflakeApi('the_account',
'the_warehouse',
'the_database',
'the_user',
'the_pwd')
api.get_connection()
mock_connect.assert_called_once_with(account='account_url', # Will be the output of self._account_url()
user='the_user',
password='the_pwd',
warehouse='the_warehouse',
database='the_database')
If you're testing other classes that use your SnowFlakeApi wrapper, then you should use the same approach, but patch the SnowFlakeApi itself in those tests.
from package.module.SomeClassThatUsesSnowFlakeApi
class TestSomeClassThatUsesSnowFlakeApi(TestCase):
#patch('package.module.SnowFlakeApi')
def test_some_func(self, mock_api):
instance = SomeClassThatUsesSnowFlakeApi()
instance.do_something()
mock_api.assert_called_once_with(...)
mock_api.return_value.get_connection.assert_called_once_with()
Also note that if you're using Python 2, you will need to pip install mock and then from mock import patch.
Using stubbing and dependency injection
from ... import SnowflakeApi
def some_func(*args, api=None, **kwargs):
api = api or SnowflakeApi(...)
conn = api.get_connection()
# Do some work
return result
Your test
class SnowflakeApiStub(SnowflakeApi)
def __init__(self):
# bypass super constructor
self.__connection = MagicMock()
def test_some_func():
stub = SnowflakeApiStub()
mock_connection = stub.__connection
mock_cursor = mock_connection.cursor.return_value
expect = ...
actual = some_func(api=stub)
assert expect == actual
assert mock_cursor.execute.called
An example using cursor, execute, and fetchone.
import snowflake.connector
class AlongSamePolly:
def __init__(self, conn):
self.conn = conn
def row_count(self):
cur = self.conn.cursor()
query = cur.execute('select count(*) from schema.table;')
return query.fetchone()[0] # returns (12345,)
# I like to dependency inject the snowflake connection object in my classes.
# This lets me use Snowflake Python Connector's built in context manager to
# rollback any errors and automatically close connections. Then you don't have
# try/except/finally blocks everywhere in your code.
#
if __name__ == '__main__':
with snowflake.connector.connect(user='user', password='password') as con:
same = AlongSamePolly(con)
print(same.row_count())
# => 12345
In the unittests you mock out the expected method calls - cursor(), execute(),
fetchone() and define the return value to follow up the chain of defined mocks.
import unittest
from unittest import mock
from along_same_polly import AlongSamePolly
class TestAlongSamePolly(unittest.TestCase):
def test_row_count(self):
with mock.patch('snowflake.connector.connect') as mock_snowflake_conn:
mock_query = mock.Mock()
mock_query.fetchone.return_value = (123,)
mock_cur = mock.Mock()
mock_cur.execute.return_value = mock_query
mock_snowflake_conn.cursor.return_value = mock_cur
same = AlongSamePolly(mock_snowflake_conn)
self.assertEqual(same.row_count(), 123)
if __name__ == '__main__':
unittest.main()
The following Solution Worked for me.
def test_connect(env_var_setup, monkeypatch):
monkeypatch.setattr(snowflake.connector.connection.SnowflakeConnection,
"connect", mocked_sf_connect
)
# calling snowflake connector method
file_job_map(env_var_setup).connect()
#mocked connection
def mocked_sf_connect(self, **kwargs):
print("Connection Successfully Established")
return True