I need to test loads of functions that used a sqlite db. To get started I want to use pytest for fixtures.
conftest.py:
import pytest
import sqlite3
#pytest.fixture
def session():
connection = sqlite3.connect(":memory:")
cursor = connection.cursor()
cursor.execute("CREATE TABLE Investment (ID INTEGER PRIMARY KEY, name VARCHAR(120), ticker VARCHAR(10), exchange VARCHAR(10), type INTEGER, relativeAddress VARCHAR(50), sharesiesFundID VARCHAR(36))")
cursor.execute("CREATE TABLE Orders (investmentID INTEGER NOT NULL, logTimestamp TIMESTAMP NOT NULL, amount INTEGER, PRIMARY KEY(investmentID, logTimestamp), FOREIGN KEY(investmentID) REFERENCES Investment(ID))")
cursor.execute("CREATE TABLE InvestmentType (typeID INTEGER NOT NULL PRIMARY KEY , entryName VARCHAR(20))")
cursor.execute("INSERT INTO InvestmentType (typeID, entryName) VALUES (0, 'Company'), (1, 'ETF'), (2, 'Managed Fund')")
cursor.execute("INSERT INTO Investment (name, ticker, exchange, type, relativeAddress, sharesiesFundID) VALUES ('3M Co.', 'MMM', 'NYSE', 0, 'nyse-mmm', '94de52ef-324f-4d24-8a80-a5d2f00656bf'), ('a2 Milk Company', 'ATM', 'NZX', 0, 'atm', 'deff31bd-625b-4a82-bbc2-064c7b70b97c'), ('Abbott Laboratories', 'ABT', 'NYSE', 0, 'nyse-abt', 'a367613c-a9bd-4562-a8fd-459e7bd4f5ae')")
connection.commit()
yield cursor
connection.close()
test_db.py:
def get_entry(session):
result = session.execute("SELECT name FROM Investment WHERE ID = 3").fetchone()
assert result[0][1] == 'Abbott Laboratories'
this keep resulting in "Process finished with exit code 5".
I've tried putting everything in the same file, and some other configurations for pytest.
Related
By given table creation statement and query it's necessary to get old values before update:
CREATE TABLE IF NOT EXISTS products(
id INT GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY,
product_id INT UNIQUE,
image_link CHARACTER VARYING NOT NULL,
additional_image_links CHARACTER VARYING[] NOT NULL
);
WITH temp AS (
INSERT INTO products(product_id, image_link, additional_image_links)
VALUES(1, 'http://www.e1xazm1ple1k113.com',ARRAY['http://www.examkple1113.com','http://www.example2.com'])
ON CONFLICT (product_id) DO UPDATE SET image_link = EXCLUDED.image_link, additional_image_links = EXCLUDED.additional_image_links
WHERE products.image_link != EXCLUDED.image_link OR products.additional_image_links != EXCLUDED.additional_image_links OR products.image_link != EXCLUDED.image_link
RETURNING id, image_link, additional_image_links
)
SELECT image_link, additional_image_links FROM products WHERE id IN (SELECT id FROM temp);
If conflict happens and new values conform criteria result is generated, however I need to use sqlalchemy machinery for it. Approximate but not working example:
def upsert(table, rows, constraint, update_cols):
query = insert(table).values(rows)
return query.on_conflict_do_update(
constraint=constraint,
set_={c: getattr(query.excluded, c) for c in update_cols},
where=getattr(table.c, "additional_image_link") != getattr(query.excluded, "additional_image_link"),
).cte("upsert")
Calling which produces the exception:
sesh = session(autocommit=False, autoflush=False, engine=DEFAULT)
sesh.execute(upsert(*args))
sqlalchemy.exc.ArgumentError: Executable SQL or text() construct expected, got <sqlalchemy.sql.selectable.CTE at 0x1042c3f10; upsert>.
i had managed to create tables in postgres but encountered issues when trying to insert values.
comands = (
CREATE TYPE student AS (
name TEXT,
id INTEGER
)
CREATE TABLE studentclass(
date DATE NOT NULL,
time TIMESTAMPTZ NOT NULL,
PRIMARY KEY (date, time),
class student
)
)
And in psycog2
command = (
INSERT INTO studentclass (date, time, student) VALUES (%s,%s, ROW(%s,%s)::student)
)
student_rec = ("John", 1)
record_to_insert = ("2020-05-21", "2020-05-21 08:10:00", student_rec)
cursor.execute(commands, record_to_insert)
When executed, the errors are the incorrect argument and if i tried to hard coded the student value inside the INSERT statement, it will inform me about the unrecognized column for student.
Please advise.
One issue is the column name is class not student. Second is psycopg2 does tuple adaption as composite type
So you can do:
insert_sql = "INSERT INTO studentclass (date, time, class) VALUES (%s,%s,%s)"
student_rec = ("John", 1)
record_to_insert = ("2020-05-21", "2020-05-21 08:10:00", student_rec)
cur.execute(insert_sql, record_to_insert)
con.commit()
select * from studentclass ;
date | time | class
------------+-------------------------+----------
05/21/2020 | 05/21/2020 08:10:00 PDT | (John,1)
class datalog(display_clock):
def con_mysql(self):
cat = mysql.connector.connect(
host="localhost", user="subramanya", passwd="Sureshbabu#4155", database="CFM")
if (cat):
datacursor = cat.cursor()
todaydate = d
check_table = (
"SELECT count(*) FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME=%s")
datacursor.execute(check_table, (todaydate,))
result = datacursor.fetchone()
if (result):
self.success_login()
else:
datacursor.execute(
"CREATE TABLE {today}(Sl_no INT NOT NULL AUTO_INCREMENT PRIMARY KEY,date DATE,Start_time TIME,End_time TIME,Item CHAR(255),Weight FLOAT, Amount INTEGER(10))".format(today=todaydate))
self.success_login()
else:
datacursor.Terminate
self.error_display.insert(0.0, "Connecting Database failed!!!")
I tried to check whether any table exists for today's date or not.
if not create the same.
no error occurred. But table not created for sysdate.
Welcome to stackoverflow!
I believe there is a small misconception here. You don't need to check if the table exists beforehand and create it afterward. Most of the current database technologies accept the condition IF NOT EXISTS on CREATE TABLE CLAUSE.
CREATE TABLE IF NOT EXISTS sales (
sale_id INT NOT NULL,
);
It means the table sales will be only created IF NOT EXISTS previously.
Also, I strongly recommend refactoring your code a wee bit. Take as a suggestion (please adapt accordingly your project needs):
from datetime import datetime
class Settings:
# please, avoid hard-coded credentials.
DB_HOST = "localhost"
DB_USER = "subramanya"
DB_PASSWD = "Sureshbabu#4155"
DB_SCHEMA = "CFM"
class datalog(display_clock):
def db_connect(self):
conn = mysql.connector.connect(
host=Settings.DB_HOST,
user=Settings.DB_USER,
passwd=Settings.DB_PASSWD,
database=Settings.DB_SCHEMA
)
if not conn:
raise Exception("Connecting Database failed!!!")
return conn
def ensure_table(self):
conn = self.db_connect()
conn.datacursor.execute("""
CREATE TABLE IF NOT EXISTS `{0}`(
Sl_no INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
date DATE,
Start_time TIME,
End_time TIME,
Item CHAR(255),
Weight FLOAT,
Amount INTEGER(10)
);
""".format(datetime.today().strftime('%Y%m%d')) # format 20200915
)
def run(self):
self.ensure_table()
self.success_login()
There are plenty of ways to write this code, but keep in mind that readability matters a lot.
I'm trying to migrate a Tomcat app from using Postgres 9.5 to SQL Server 2016 and I've got a problem statement I can't seem to duplicate.
It's basically an upsert but one of the complications is the request supplies arguments to do the update, but when there is conflict I need to use some of the existing values from conflicting rows to insert/update.
The primary keys in the table can sometimes cause a conflict, which requires updating rows and deleting the old ones.
The table schema in MS SQL looks like:
CREATE TABLE [dbo].[signup](
[site_key] [varchar](32) NOT NULL,
[list_id] [bigint] NOT NULL,
[email_address] [varchar](256) NOT NULL,
[customer_id] [bigint] NULL,
[attribute1] [varchar](64) NULL,
[date1] [datetime] NOT NULL,
[date2] [datetime] NULL,
CONSTRAINT [pk_signup] PRIMARY KEY CLUSTERED
(
[site_key] ASC,
[list_id] ASC,
[email_address] ASC
)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]
) ON [PRIMARY]
The old Postgres SQL looked like this:
WITH updated_rows AS (
INSERT INTO signup
(site_key, list_id, email_address, customer_id, attribute1, date1, date2)
SELECT site_key, list_id, :emailAddress, customer_id, attribute1, date1, date2
FROM signup WHERE customer_id = :customerId and email_address <> :emailAddress
ON CONFLICT (site_key, list_id, email_address) DO UPDATE SET customer_id = excluded.customer_id
RETURNING site_key, customer_id, email_address, list_id
)
DELETE FROM signup AS signup_delete USING updated_rows
WHERE
signup_delete.site_key = updated_rows.site_key
AND signup_delete.customer_id = updated_rows.customer_id
AND signup_delete.list_id = updated_rows.list_id
AND signup_delete.email_address <> :emailAddress;
Two arguments are supplied, customer id and email address, shown here as Spring NamedParameterJdbcTemplate values :customerId and :emailAddress
It's trying to change the email address of the customer id to be the supplied one, but sometimes the supplied email address already exists in the primary key constraint.
In which case it needs to change the existing customer id to be supplied one, and remove the rows with that don't match the new email address.
I also need to try and maintain isolation so that nothing can change the data whilst I'm updating.
I'm trying to do it with a MERGE statement but I can't seem to get it to work, it's complaining I cant use values that aren't in the clause scope, but I think I've probably got other issues here too.
This is what I had so far. It doesn't even address the deleting part - only the upserting, but I can't even get this part to work. I was planning to use the OUTPUT from this as input to something to delete the rows similar to the postgres version.
WITH source AS (
SELECT cs.[site_key] as existing_site_key,
cs.list_id as existing_list_id,
cs.email_address as existing_email,
cs.customer_id as existing_customer_id,
cs.attribute1 as existing_attribute1,
cs.date1 as existing_date1,
cs.date2 as existing_date2,
cs2.email_address as conflicting_email,
cs2.customer_id AS conflicting_customer_id
FROM [dbo].[signup] cs
LEFT JOIN [dbo].[signup] cs2 ON cs2.email_address = :emailAddress
AND cs.site_key = cs2.site_key
AND cs.list_id = cs2.list_id
WHERE cs.customer_id = :customerId
)
MERGE signup WITH (HOLDLOCK) AS target
USING source
ON ( source.conflicting_customer_id is not null )
WHEN MATCHED AND source.existing_site_key = target.site_key AND source.existing_list_id = target.list_id AND source.conflicting_email = target.email_address THEN UPDATE
SET customer_id = :customerId
WHEN NOT MATCHED BY target AND source.existing_site_key = target.site_key AND source.existing_list_id = target.list_id AND source.conflicting_customer_id = :customerId THEN INSERT
(site_key, list_id, email_address, customer_id, attribute1, date1, date2) VALUES
(source.existing_site_key, source.existing_list_id, :emailAddress, source.customer_id, source.existing_attribute1, source.existing_date1, source.existing_date2)
Thanks,
mikee
With the following:
engine = sqlalchemy.create_engine(url)
df = pd.DataFrame({
"eid": [1,2],
"f_i": [123, 1231],
"f_i_arr": [[123], [0]],
"f_53": ["2013/12/1","2013/12/1",],
"f_53a": [["2013/12/1"], ["2013/12/1"],],
})
with engine.connect() as con:
con.execute("""
DROP TABLE IF EXISTS public.test;
CREATE TABLE public.test
(
eid integer NOT NULL,
f_i INTEGER NULL,
f_i_arr INTEGER NULL,
f_53 DATE NULL,
f_53a DATE[] NULL,
PRIMARY KEY(eid)
);;
""")
df.to_sql("test", con, if_exists='append')
If I try to insert only column "f_53" (an date) it succeeds.
If I try to add column "f_53a" (a date[]) it fails with:
^
sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) column "f_53a" is of type date[] but expression is of type text[]
LINE 1: ..._53, f_53a, f_i, f_i_arr) VALUES (1, '2013/12/1', ARRAY['201...
^
HINT: You will need to rewrite or cast the expression.
[SQL: 'INSERT INTO test (eid, f_53, f_53a, f_i, f_i_arr) VALUES (%(eid)s, %(f_53)s, %(f_53a)s, %(f_i)s, %(f_i_arr)s)'] [parameters: ({'f_53': '2013/12/1', 'f_53a': ['2013/12/1', '2013/12/1'], 'f_i_arr': [123], 'eid': 1, 'f_i': 123}, {'f_53': '2013/12/1', 'f_53a': ['2013/12/1', '2013/12/1'], 'f_i_arr': [0], 'eid': 2, 'f_i': 1231})]
I have mentioned the dtypes explicitly and it worked for me for postgres.
//sample code
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.dialects import postgresql
df.to_sql('mytable',pgConn, if_exists='append', index=False, dtype={'datetime': sqlalchemy.TIMESTAMP(), 'cur_c':postgresql.ARRAY(sqlalchemy.types.REAL),
'volt_c':postgresql.ARRAY(sqlalchemy.types.REAL)
})
Yes -- is possible to insert [] and [][] types from a dataframe into postgres form a dataframe.
Unlike flat DATE types, which are may be correctly parsed by sql, DATE[] and DATE[][] need to be converted to datetime objects first. Like so.
with engine.connect() as con:
con.execute("""
DROP TABLE IF EXISTS public.test;
CREATE TABLE public.test
(
eid integer NOT NULL,
f_i INTEGER NULL,
f_ia INTEGER[] NULL,
f_iaa INTEGER[][] NULL,
f_d DATE NULL,
f_da DATE[] NULL,
f_daa DATE[][] NULL,
PRIMARY KEY(eid)
);
""")
d = pd.to_datetime("2013/12/1")
i = 99
df = pd.DataFrame({
"eid": [1,2],
"f_i": [i,i],
"f_ia": [None, [i,i]],
"f_iaa": [[[i,i],[i,i]], None],
"f_d": [d,d],
"f_da": [[d,d],None],
"f_daa": [[[d,d],[d,d]],None],
})
df.to_sql("test", con, if_exists='append', index=None)