Related
I'm learning python... I tried to execute sql queries with parameters without success...
I tried:
from tkinter import*
import tkinter as tk
from tkinter import ttk
import sqlite3
realNumber = 2.0
database = sqlite3.connect('NumDB.db')
cursor = database.cursor()
cursor.execute("SELECT numColumn from numTable WHERE realNumber=?", ( realNumber ))
results = cursor.fetchall()
print(results)
cursor.close()
database.close()
it works when I define the variable as text:
realNumber "2.0"
or
cursor.execute("SELECT numColumn from numTable WHERE realNumber=?", ( str(realNumber)))
the type of realNumber is set on real in the database.
is it possible to use real or integer variables without converting to string?
Thanks
Depending on the version of Python, sample code below don't require a type conversion: (Note: to avoid sql injection option 4 is best.)
def func1(ag):
return f" variable {ag}"
def func2(ag):
return "SELECT numColumn from numTable WHERE realNumber=?", ( ag )
def func3(ag):
return f"SELECT numColumn from numTable WHERE realNumber={ag}"
def func4(ag):
sql = "SELECT numColumn from numTable WHERE realNumber={}"
return sql.format(ag)
ag = 40.5
print(func1(ag))
print(func2(ag))
print(func3(ag))
print(func4(ag))
Output:
variable 40.5
('SELECT numColumn from numTable WHERE realNumber=?', 40.5)
SELECT numColumn from numTable WHERE realNumber=40.5
SELECT numColumn from numTable WHERE realNumber=40.5
#---------------Original code in question-----------------
from tkinter import*
import tkinter as tk
from tkinter import ttk
import sqlite3
realNumber = 2.0
database = sqlite3.connect('NumDB.db')
cursor = database.cursor()
cursor.execute("SELECT numColumn from numTable WHERE realNumber={}".format(realNumber) )
results = cursor.fetchall()
print(results)
cursor.close()
database.close()
I was tinkering around with the INSERT INTO example from this post:
INSERT example:
import boto3
sql = """
INSERT INTO YOUR_TABLE_NAME_HERE
(
your_column_name_1
,your_column_name_2
,your_column_name_3)
VALUES(
:your_param_1_name
,:your_param_2_name)
,:your_param_3_name
"""
param1 = {'name':'your_param_1_name', 'value':{'longValue': 5}}
param2 = {'name':'your_param_2_name', 'value':{'longValue': 63}}
param3 = {'name':'your_param_3_name', 'value':{'stringValue': 'para bailar la bamba'}}
param_set = [param1, param2, param3]
db_clust_arn = 'your_db_cluster_arn_here'
db_secret_arn = 'your_db_secret_arn_here'
rds_data = boto3.client('rds-data')
response = rds_data.execute_statement(
resourceArn = db_clust_arn,
secretArn = db_secret_arn,
database = 'your_database_name_here',
sql = sql,
parameters = param_set)
print(str(response))
I quickly realized that I wasn't sure how to handle composite data types. For example, suppose that I have a composite data type of the form (integer, box). How do I specify its value for the parameters arg of boto3.client('rds-data').execute_statement()?
I wasn't able to find anything in these boto3 docs
I have a Flask app that parses a CSV of public election data and inserts the results into a Postgres database. It's a port of an old, not-Flask, Python 2 app that uses various libraries that no longer work. I'm mostly trying to base the application's structure on this tutorial. I've been using Flask-SQLAlchemy to construct some models for the database tables and populate the data from the CSV.
In this case I'm working with an Area model, which corresponds to a geographic area that might have an election (house district, school board district, etc). Here's what I've got in my basic blueprint route:
election = None
#bp.route('/areas')
def scrape_areas():
area = Area()
sources = area.read_sources()
election = area.set_election()
if election not in sources:
return
# Get metadata about election
election_meta = sources[election]['meta'] if 'meta' in sources[election] else {}
for i in sources[election]:
source = sources[election][i]
if 'type' in source and source['type'] == 'areas':
rows = area.parse_election(source, election_meta)
count = 0
for row in rows:
parsed = area.parser(row, i)
area = Area()
area.from_dict(parsed, new=True)
# this shows the generated string of area_id
# which is a UNIQUE key in the database
print(area)
db.session.add(area)
db.session.commit()
count = count + 1
return count
And here's the models.py:
import logging
import os
import json
import re
import csv
import urllib.request
import calendar
import datetime
from flask import current_app
from app import db
LOG = logging.getLogger(__name__)
scraper_sources_inline = None
class ScraperModel(object):
nonpartisan_parties = ['NP', 'WI', 'N P']
def __init__(self, group_type = None):
"""
Constructor
"""
# this is where scraperwiki was creating and connecting to its database
# we do this in the imported sql file instead
self.read_sources()
def read_sources(self):
"""
Read the scraper_sources.json file.
"""
if scraper_sources_inline is not None:
self.sources = json.loads(scraper_sources_inline)
else:
#sources_file = current_app.config['SOURCES_FILE']
sources_file = os.path.join(current_app.root_path, '../scraper_sources.json')
data = open(sources_file)
self.sources = json.load(data)
return self.sources
def set_election(self):
# Get the newest set
newest = 0
for s in self.sources:
newest = int(s) if int(s) > newest else newest
newest_election = str(newest)
election = newest_election
# Usually we just want the newest election but allow for other situations
election = election if election is not None and election != '' else newest_election
return election
def parse_election(self, source, election_meta = {}):
# Ensure we have a valid parser for this type
parser_method = getattr(self, "parser", None)
if callable(parser_method):
# Check if election has base_url
source['url'] = election_meta['base_url'] + source['url'] if 'base_url' in election_meta else source['url']
# Get data from URL
try:
response = urllib.request.urlopen(source['url'])
lines = [l.decode('latin-1') for l in response.readlines()]
rows = csv.reader(lines, delimiter=';')
return rows
except Exception as err:
LOG.exception('[%s] Error when trying to read URL and parse CSV: %s' % (source['type'], source['url']))
raise
def from_dict(self, data, new=False):
for field in data:
setattr(self, field, data[field])
class Area(ScraperModel, db.Model):
__tablename__ = "areas"
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
area_id = db.Column(db.String(255), unique=True, nullable=False)
areas_group = db.Column(db.String(255))
county_id = db.Column(db.String(255))
county_name = db.Column(db.String(255))
ward_id = db.Column(db.String(255))
precinct_id = db.Column(db.String(255))
precinct_name = db.Column(db.String(255))
state_senate_id = db.Column(db.String(255))
state_house_id = db.Column(db.String(255))
county_commissioner_id = db.Column(db.String(255))
district_court_id = db.Column(db.String(255))
soil_water_id = db.Column(db.String(255))
school_district_id = db.Column(db.String(255))
school_district_name = db.Column(db.String(255))
mcd_id = db.Column(db.String(255))
precincts = db.Column(db.String(255))
name = db.Column(db.String(255))
updated = db.Column(db.DateTime, default=db.func.current_timestamp(), onupdate=db.func.current_timestamp())
def __repr__(self):
return '<Area {}>'.format(self.area_id)
def parser(self, row, group):
# General data
parsed = {
'area_id': group + '-',
'areas_group': group,
'county_id': None,
'county_name': None,
'ward_id': None,
'precinct_id': None,
'precinct_name': '',
'state_senate_id': None,
'state_house_id': None,
'county_commissioner_id': None,
'district_court_id': None,
'soil_water_id': None,
'school_district_id': None,
'school_district_name': '',
'mcd_id': None,
'precincts': None,
'name': ''
}
if group == 'municipalities':
parsed['area_id'] = parsed['area_id'] + row[0] + '-' + row[2]
parsed['county_id'] = row[0]
parsed['county_name'] = row[1]
parsed['mcd_id'] = "{0:05d}".format(int(row[2])) #enforce 5 digit
parsed['name'] = row[1]
if group == 'counties':
parsed['area_id'] = parsed['area_id'] + row[0]
parsed['county_id'] = row[0]
parsed['county_name'] = row[1]
parsed['precincts'] = row[2]
if group == 'precincts':
parsed['area_id'] = parsed['area_id'] + row[0] + '-' + row[1]
parsed['county_id'] = row[0]
parsed['precinct_id'] = row[1]
parsed['precinct_name'] = row[2]
parsed['state_senate_id'] = row[3]
parsed['state_house_id'] = row[4]
parsed['county_commissioner_id'] = row[5]
parsed['district_court_id'] = row[6]
parsed['soil_water_id'] = row[7]
parsed['mcd_id'] = row[8]
if group == 'school_districts':
parsed['area_id'] = parsed['area_id'] + row[0]
parsed['school_district_id'] = row[0]
parsed['school_district_name'] = row[1]
parsed['county_id'] = row[2]
parsed['county_name'] = row[3]
return parsed
So Areas is an extension of my default model class because it allows me to set up the fields that are specific to a given area based on the CSV.
Where this code fails is a (relatively) rare case in the CSV data where there might be multiple rows that, in the old application, correspond to the same row in the table. That old application had an array (usually with just one item, representing a UNIQUE column in the database) to instruct the code to run an UPDATE on those rows.
It returns an error like this:
sqlalchemy.exc.IntegrityError: (psycopg2.errors.UniqueViolation) duplicate key value violates unique constraint "areas_area_id_key"
DETAIL: Key (area_id)=(counties-01) already exists.
An example of how it runs when I'm just logging my UNIQUE key value from the model instead of inserting it:
<Area precincts-87-0140>
<Area precincts-87-0145>
<Area precincts-87-0150>
<Area precincts-87-0155>
<Area precincts-87-0160>
<Area precincts-87-0165>
<Area school_districts-0001>
<Area school_districts-0001>
<Area school_districts-0001>
<Area school_districts-0002>
<Area school_districts-0004>
<Area school_districts-0006>
<Area school_districts-0012>
<Area school_districts-0013>
<Area school_districts-0014>
So I've been looking at different methods that Flask can use to run an UPSERT statement because I'd need to update all of the fields, and they'd be different based on both which type of area it is, and also in the other models (election contests or results, for example). Most of what I'm finding uses SQLAlchemy rather than Flask-SQLAlchemy.
I found this answer that looked promising. Here's what I added to my model:
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.sql.expression import Insert
Then I modified the ScraperModel class like this:
class ScraperModel(object):
#compiles(Insert)
def compile_upsert(insert_stmt, compiler, **kwargs):
"""
converts every SQL insert to an upsert i.e;
INSERT INTO test (foo, bar) VALUES (1, 'a')
becomes:
INSERT INTO test (foo, bar) VALUES (1, 'a') ON CONFLICT(foo) DO UPDATE SET (bar = EXCLUDED.bar)
(assuming foo is a primary key)
:param insert_stmt: Original insert statement
:param compiler: SQL Compiler
:param kwargs: optional arguments
:return: upsert statement
"""
pk = insert_stmt.table.primary_key
insert = compiler.visit_insert(insert_stmt, **kwargs)
ondup = f'ON CONFLICT ({",".join(c.name for c in pk)}) DO UPDATE SET'
updates = ', '.join(f"{c.name}=EXCLUDED.{c.name}" for c in insert_stmt.table.columns)
upsert = ' '.join((insert, ondup, updates))
return upsert
I'm clearly misunderstanding how the insert_stmt works because of how the query comes out, but here's the error that it generates:
sqlalchemy.exc.ProgrammingError: (psycopg2.errors.SyntaxError) syntax error at or near "ON"
LINE 1: ..., '54', '', CURRENT_TIMESTAMP) RETURNING areas.id ON CONFLIC...
^
[SQL: INSERT INTO areas (area_id, areas_group, county_id, county_name, ward_id, precinct_id, precinct_name, state_senate_id, state_house_id, county_commissioner_id, district_court_id, soil_water_id, school_district_id, school_district_name, mcd_id, precincts, name, updated) VALUES (%(area_id)s, %(areas_group)s, %(county_id)s, %(county_name)s, %(ward_id)s, %(precinct_id)s, %(precinct_name)s, %(state_senate_id)s, %(state_house_id)s, %(county_commissioner_id)s, %(district_court_id)s, %(soil_water_id)s, %(school_district_id)s, %(school_district_name)s, %(mcd_id)s, %(precincts)s, %(name)s, CURRENT_TIMESTAMP) RETURNING areas.id ON CONFLICT (id) DO UPDATE SET id=EXCLUDED.id, area_id=EXCLUDED.area_id, areas_group=EXCLUDED.areas_group, county_id=EXCLUDED.county_id, county_name=EXCLUDED.county_name, ward_id=EXCLUDED.ward_id, precinct_id=EXCLUDED.precinct_id, precinct_name=EXCLUDED.precinct_name, state_senate_id=EXCLUDED.state_senate_id, state_house_id=EXCLUDED.state_house_id, county_commissioner_id=EXCLUDED.county_commissioner_id, district_court_id=EXCLUDED.district_court_id, soil_water_id=EXCLUDED.soil_water_id, school_district_id=EXCLUDED.school_district_id, school_district_name=EXCLUDED.school_district_name, mcd_id=EXCLUDED.mcd_id, precincts=EXCLUDED.precincts, name=EXCLUDED.name, updated=EXCLUDED.updated]
[parameters: {'area_id': 'counties-01', 'areas_group': 'counties', 'county_id': '01', 'county_name': 'Aitkin', 'ward_id': None, 'precinct_id': None, 'precinct_name': '', 'state_senate_id': None, 'state_house_id': None, 'county_commissioner_id': None, 'district_court_id': None, 'soil_water_id': None, 'school_district_id': None, 'school_district_name': '', 'mcd_id': None, 'precincts': '54', 'name': ''}]
(Background on this error at: https://sqlalche.me/e/14/f405)
I'm hoping I didn't paste too much to be helpful there.
I also found this answer that I read as creating its own insert statement instead of compiling the built in one. Here's what I changed. In the blueprint's imports:
from sqlalchemy.dialects.postgresql import insert
And in the blueprint's loop:
for i in sources[election]:
source = sources[election][i]
if 'type' in source and source['type'] == 'areas':
rows = area.parse_election(source, election_meta)
count = 0
for row in rows:
parsed = area.parser(row, i)
area = Area()
area.from_dict(parsed, new=True)
stmt = insert(Area.__table__).values(parsed)
stmt = stmt.on_conflict_do_update(
# Let's use the constraint name which was visible in the original posts error msg
constraint="['area_id']",
# The columns that should be updated on conflict
set_={
parsed
}
)
db.session.execute(stmt)
count = count + 1
return count
It results in a different error:
TypeError: unhashable type: 'dict'
All that to say, I'm currently at a loss. It's clear to me that I need to modify the INSERT statement, but it's not clear to me which route I should take to modify it, how to make sure that it matches on the correct field (which is called area_id and the key is called areas_id_unique), or how to make sure it updates the correct fields when it does find a match.
What I think I'm finding is that none of this would work because I wasn't matching on a primary key, but on a unique key. What I've done is change the unique key area_id to a primary key. Then, I can use the upsert statement from above.
#compiles(Insert)
def compile_upsert(insert_stmt, compiler, **kwargs):
"""
converts every SQL insert to an upsert i.e;
INSERT INTO test (foo, bar) VALUES (1, 'a')
becomes:
INSERT INTO test (foo, bar) VALUES (1, 'a') ON CONFLICT(foo) DO UPDATE SET (bar = EXCLUDED.bar)
(assuming foo is a primary key)
:param insert_stmt: Original insert statement
:param compiler: SQL Compiler
:param kwargs: optional arguments
:return: upsert statement
"""
pk = insert_stmt.table.primary_key
insert = compiler.visit_insert(insert_stmt, **kwargs)
ondup = f'ON CONFLICT ({",".join(c.name for c in pk)}) DO UPDATE SET'
updates = ', '.join(f"{c.name}=EXCLUDED.{c.name}" for c in insert_stmt.table.columns)
upsert = ' '.join((insert, ondup, updates))
return upsert
I had been trying to change the pk = insert_stmt.table.primary_key line to check for the unique key with no success, but it works just like this if I change that field.
Changing the primary key also fixed the other solution I was trying:
group = []
for row in rows:
parsed = area.parser(row, i)
area = Area()
area.from_dict(parsed, new=True)
group.append(area)
insert(db.session, Area, group)
def insert(session, model, rows):
table = model.__table__
stmt = insert(table)
primary_keys = [key.name for key in inspect(table).primary_key]
update_dict = {c.name: c for c in stmt.excluded if not c.primary_key}
if not update_dict:
raise ValueError("insert_or_update resulted in an empty update_dict")
stmt = stmt.on_conflict_do_update(
index_elements=primary_keys,
set_=update_dict
)
So both solutions were (relatively) workable, but only with a primary key instead of a unique key and that just hadn't been clear to me.
I'm attempting to collect options data using a heroku dyno (hobby $7 version) and a heroku Postgres database (standard $50 version). I've got a small script set up that uses yfinance to collect options data for a set of ticker symbols and SQLAlchemy + psycopg2 to insert this data into the postgres database.
I seem to be encountering a very strange error when inserting into the database. For many of the records, I'm seeing a (psycopg2.errors.NumericValueOutOfRange) integer out of range Exception thrown for records that should not have any integers out of range... I've included an example of the error below as well as my model definition. At first I thought maybe it was the id / pk column, but according to SQLAlchemy's docs they automatically treat these fields as a serial data type. So I'm not sure why this error is being thrown with all integer columns clearly in the range of the INT data type.
Is this a heroku thing? Has anyone seen / dealt with this before? I'm not sure how to debug the underlying issue. Thanks in advance!
2021-03-24 12:25:16 ERROR Unable to commit Option(id=None, ticker=EDIT, option_type=PUT, contractSymbol=EDIT230120P00017500, lastTradeDate=2020-11-05 14:38:12, strike=17.5, lastPrice=4.7, bid=0.0, ask=0.0, change=0.0, percentChange=0.0, volume=1.0, openInterest=0, impliedVolatility=0.12500875, inTheMoney=False, contractSize=REGULAR, currency=USD)
2021-03-24 12:25:16 ERROR This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). Original exception was: (psycopg2.errors.NumericValueOutOfRange) integer out of range
(Background on this error at: http://sqlalche.me/e/14/9h9h) (Background on this error at: http://sqlalche.me/e/14/7s2a)
Continuing.
Model definition:
class Option(config.Base):
__tablename__ = "options"
id = Column(Integer, primary_key=True)
ticker = Column(String, nullable=False)
dt_created_db = Column(
DateTime, nullable=False, default=datetime.datetime.utcnow()
)
dt_updated_db = Column(
DateTime, nullable=False, default=datetime.datetime.utcnow()
)
dt_created_market_time_db = Column(
DateTime, nullable=False,
default=datetime.datetime.now().astimezone(
pytz.timezone(
"America/New_York"
)
)
)
option_type = Column(String, nullable=False) # CALL or PUT
contractSymbol = Column(String, nullable=False)
lastTradeDate = Column(DateTime, nullable=False)
strike = Column(Float, nullable=False)
lastPrice = Column(Float, nullable=False)
bid = Column(Float)
ask = Column(Float)
change = Column(Float)
percentChange = Column(Float)
volume = Column(Float)
openInterest = Column(Integer)
impliedVolatility = Column(Numeric)
inTheMoney = Column(Boolean)
contractSize = Column(String)
currency = Column(String)
#classmethod
def create_option_record(cls, ticker, record, opt_type):
"""Creates an Option model instance.
Args:
ticker: The ticker symbol.
record: A pandas dataframe row with "." access to columns.
opt_type: A string, either CALL or PUT.
Returns:
Option object.
"""
return cls(
ticker=ticker,
option_type=opt_type,
contractSymbol=record.contractSymbol,
lastTradeDate=record.lastTradeDate,
strike=record.strike,
lastPrice=record.lastPrice,
bid=record.bid,
ask=record.ask,
change=record.change,
percentChange=record.percentChange,
volume=record.volume,
openInterest=record.openInterest,
impliedVolatility=round(record.impliedVolatility, 20),
inTheMoney=record.inTheMoney,
contractSize=record.contractSize,
currency=record.currency
)
def __str__(self):
return (
f"Option("
f"id={self.id}, "
f"ticker={self.ticker}, "
f"option_type={self.option_type}, "
f"contractSymbol={self.contractSymbol}, "
f"lastTradeDate={self.lastTradeDate}, "
f"strike={self.strike}, "
f"lastPrice={self.lastPrice}, "
f"bid={self.bid}, "
f"ask={self.ask}, "
f"change={self.change}, "
f"percentChange={self.percentChange}, "
f"volume={self.volume}, "
f"openInterest={self.openInterest}, "
f"impliedVolatility={self.impliedVolatility}, "
f"inTheMoney={self.inTheMoney}, "
f"contractSize={self.contractSize}, "
f"currency={self.currency}"
f")"
)
I've solved this. It turns out the database logs were behind the compute logs and so the record I thought was causing the error was not actually causing the error.
The root cause was that an np.nan value was being inserted into an INTEGER column. I had to replace np.nan's with None and this solved the problem.
I am setting up a DirectQuery connection from my local MongoDB environment to Microsoft Power BI. I created a custom connector using the sample ODBC connector from Microsoft (https://github.com/microsoft/DataConnectors/tree/master/samples). I am using the latest MongoDB ODBC Driver and MongoDB ODBC Driver for BI Connector. I created a System Data Source that is used as the input for my custom connector.
I am bringing in the following collection to Power BI for DirectQuery:
tbllinktrafficdata. I try to create a Slicer with the LastUpdate field. When I filter on one LinkID (which is a field), I get the following error.
"ErrorMessage":"Failed to convert type bigint to integer, expression `LinkID` to 1506780."
"ErrorMessage":"This ODBC driver doesn't support SQL_FN_CVT_CONVERT or SQL_FN_CVT_CAST. You can override this by using SqlGetInfo for SQL_CONVERT_FUNCTIONS."
It looks like I'm seeing a conversion error, but I'm not sure how to resolve the issue within the connector.
Here is my SqlGetInfo function:
SQLGetInfo = [
// place custom overrides here
SQL_SQL92_PREDICATES = ODBC[SQL_SP][All],
SQL_AGGREGATE_FUNCTIONS = ODBC[SQL_AF][All],
SQL_CONVERT_FUNCTIONS = ODBC[SQL_CVT][BIGINT]
],
//this is from OdbcConstants file that is called in Power Query file
SQL_CVT = [
BIGINT = 0x00004000
]
Here is more of the Power BI Trace Log:
SqlTranslator/SqlParser/Parse {"Start":"2019-09-
03T19:55:54.6245555Z","Action":"SqlTranslator/SqlParser/Parse","HostProcessId":"12228","SQL":"\nSELECT MAX([t29].[LastUpdate])\n AS [a0],MIN([t29].[LastUpdate])\n AS [a1]\nFROM \n(\n(SELECT * FROM [tbllinktrafficdata (2)])\n)\n AS [t29]\nWHERE \n(\n[t29].[LinkID] = 1506780\n)\n ","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.1356012"}
SqlExpressionTranslator/Translate {"Start":"2019-09-03T19:55:54.7606768Z","Action":"SqlExpressionTranslator/Translate","HostProcessId":"12228","IsRecognized":"True","Result":"(environment) => Table.RenameColumns(let\r\n t1133 = Table.RenameColumns(Table.PrefixColumns(environment[#\"tbllinktrafficdata (2)\"], \"tbllinktrafficdata (2)\"), {{\"tbllinktrafficdata (2)._id\", \"t29._id\"}, {\"tbllinktrafficdata (2).AgencyID\", \"t29.AgencyID\"}, {\"tbllinktrafficdata (2).DataType\", \"t29.DataType\"}, {\"tbllinktrafficdata (2).LastUpdate\", \"t29.LastUpdate\"}, {\"tbllinktrafficdata (2).LinkID\", \"t29.LinkID\"}, {\"tbllinktrafficdata (2).Occupancy\", \"t29.Occupancy\"}, {\"tbllinktrafficdata (2).Speed\", \"t29.Speed\"}, {\"tbllinktrafficdata (2).TravelTime\", \"t29.TravelTime\"}, {\"tbllinktrafficdata (2).Volume\", \"t29.Volume\"}}),\r\n t1135 = Table.SelectRows(t1133, (t1134) => Value.NullableEquals(t1134[t29.LinkID], 1506780)),\r\n t1140 = Table.Group(t1135, {}, {{\"a0\", (t1136) => List.Max(t1136[t29.LastUpdate])}, {\"a1\", (t1137) => List.Min(t1137[t29.LastUpdate])}}),\r\n t1141 = Table.SelectColumns(t1140, {\"a0\", \"a1\"})\r\nin\r\n t1141, {{\"a0\", \"a0\"}, {\"a1\", \"a1\"}})","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.1279102"}
SimpleDocumentEvaluator/GetResult/Evaluate {"Start":"2019-09-03T19:55:54.6208186Z","Action":"SimpleDocumentEvaluator/GetResult/Evaluate","HostProcessId":"12228","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.2835266"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9495926Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","Function Name":"VisitInvocation","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0000162"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9498087Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","Function Name":"VisitValueEqualsShared","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0000058"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9498281Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","ErrorMessage":"Failed to convert type bigint to integer, expression `LinkID` to 1506780.","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0057948"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9498205Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","Function Name":"AdjustForCompatibility","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0000034"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9556351Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","Function Name":"AdjustNumberValuesToPreventOverflow","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0000045"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9556433Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","Function Name":"SoftConvertSeries","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0000034"}
OdbcQuery/FoldingWarning {"Start":"2019-09-03T19:55:54.9556496Z","Action":"OdbcQuery/FoldingWarning","HostProcessId":"12228","ErrorMessage":"This ODBC driver doesn't support SQL_FN_CVT_CONVERT or SQL_FN_CVT_CAST. You can override this by using SqlGetInfo for SQL_CONVERT_FUNCTIONS.","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0002851"}
OdbcQueryDomain/ReportFoldingFailure {"Start":"2019-09-03T19:55:54.9566845Z","Action":"OdbcQueryDomain/ReportFoldingFailure","HostProcessId":"12228","Exception":"Exception:\r\nExceptionType: Microsoft.Mashup.Engine1.Runtime.FoldingFailureException, Microsoft.MashupEngine, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35\r\nMessage: Folding failed. Please take a look the information in the trace.\r\nStackTrace:\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.CallConvertOrCast(SqlExpression from, OdbcTypeMap toType)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.TryVisitConvert(OdbcTypeInfo fromType, OdbcTypeInfo toType, SqlExpression expression, SqlExpression& convertedExpression)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.TryConvert(OdbcTypeInfo typeInfo, OdbcScalarExpression expression, OdbcScalarExpression& convertedExpression)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.<SoftConvertSeries>d__191.MoveNext()\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.AdjustNumberValuesToPreventOverflow(OdbcScalarExpression left, OdbcScalarExpression right, Precision precision)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.AdjustNumberValuesForCompatibility(OdbcScalarExpression left, OdbcScalarExpression right, Precision precision)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.AdjustForCompatibility(OdbcScalarExpression left, OdbcScalarExpression right, Precision precision)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.VisitEquals(OdbcSqlExpression leftExpression, OdbcSqlExpression rightExpression, Precision precision, Boolean nullable)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.VisitValueEqualsShared(InvocationQueryExpression expression, Boolean nullable)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQueryExpressionVisitor.VisitInvocation(InvocationQueryExpression expression)\r\n at Microsoft.Mashup.Engine1.Library.Odbc.OdbcQuery.SelectRows(FunctionValue function)\r\n\r\n\r\n","ProductVersion":"2.72.5556.801 (19.08)","ActivityId":"3ef90ded-79fc-4726-83bb-c98d8ccf5be7","Process":"Microsoft.Mashup.Container.NetFX45","Pid":22304,"Tid":1,"Duration":"00:00:00.0036644"}
I was able to convert by transforming in SqlColumn function. Odbc SQL types taken from ODBC constants SQL_TYPE (included below)
SQLColumns = (catalogName, schemaName, tableName, columnName, source) =>
let
OdbcSqlType.BIG_INT = -5,
OdbcSqlType.INTEGER = 4,
FixDataType = (dataType) =>
if dataType = OdbcSqlType.BIG_INT then
OdbcSqlType.INTEGER
else
dataType,
Transform = Table.TransformColumns(source, { { "DATA_TYPE", FixDataType } })
in
// the if statement conditions will force the values to evaluated/written to diagnostics
if (Diagnostics.LogValue("SQLColumns.TableName", tableName) <> "***" and Diagnostics.LogValue("SQLColumns.ColumnName", columnName) <> "***") then
let
// Outputting the entire table might be too large, and result in the value being truncated.
// We can output a row at a time instead with Table.TransformRows()
rows = Table.TransformRows(Transform, each Diagnostics.LogValue("SQLColumns", _)),
toTable = Table.FromRecords(rows)
in
Value.ReplaceType(toTable, Value.Type(Transform))
else
Transform,
SQL Data Types
SQL_TYPE =
[
// Base data types (sql.h)
UNKNOWN = 0,
NULL = 0,
CHAR = 1,
NUMERIC = 2,
DECIMAL = 3,
INTEGER = 4,
SMALLINT = 5,
FLOAT = 6,
REAL = 7,
DOUBLE = 8,
DATETIME = 9, // V3 Only
VARCHAR = 12,
// Unicode types (sqlucode.h)
WCHAR = -8,
WVARCHAR = -9,
WLONGVARCHAR = -10,
// Extended data types (sqlext.h)
INTERVAL = 10, // V3 Only
TIME = 10,
TIMESTAMP = 11,
LONGVARCHAR = -1,
BINARY = -2,
VARBINARY = -3,
LONGVARBINARY = -4,
BIGINT = -5,
TINYINT = -6,
BIT = -7,
GUID = -11, // V3 Only
// One-parameter shortcuts for date/time data types.
TYPE_DATE = 91,
TYPE_TIME = 92,
TYPE_TIMESTAMP = 93,
// SQL Server Types -150 to -159 (sqlncli.h)
SS_VARIANT = -150,
SS_UDT = -151,
SS_XML = -152,
SS_TABLE = -153,
SS_TIME2 = -154,
SS_TIMESTAMPOFFSET = -155
],