QGIS Processing Script Corrupts Accented Characters - qgis

Question

I am using a processing script to refactor fields for three different input layers and save the refactored results to numerous output types (.shp, .tab, .xlsx)
The input layers have string values that contain accents (French). However, the output layers end up corrupted.
I have looked at another post in which they talk about .cpg files and changing the default for OGR. I have done that but the data still comes out wrong, though it is different than if I didn't add the code to implicitly get OGR to work in utf-8.
My source layers are in UTF-8.
My output layers appear to be in UTF-8.
I am pasting my code below. Please note that I use the processing.run() to execute native:saveFeatures which I copied from the history for manually running the Processing Tool "Save Vector Features To File". When I run that tool, or the Refactor Fields tool, the output does have the correct accents in the data.
PLEASE HELP !
# -*- coding: utf-8 -*-
import os
# Section removed to adhere to stackOverflow char limit
# This section had all the import statements
orgEncoding=QgsSettings().value('/Processing/encoding') # save setting
QgsSettings().setValue('/Processing/encoding', 'utf-8') # set uft8
# Section removed to adhere to stackOverflow char limit
# This section had all the required QGIS processing script function calls
# This section also had the initAlgorithm
def processAlgorithm(self, parameters, context, feedback):
"""
Here is where the processing itself takes place.
"""
global renamer
sdcFileName = 'Eligible_Satellite_Dependent_Communities'
etcFileName = 'Eligible_Transport_Communities'
roadsFileName = 'Eligible_Roads'
sdcFileName_Fr = 'Collectivités_dépendantes_des_satellites_admissibles'
etcFileName_Fr = 'Collectivités_admissibles_sans_transport_à_grande_capacité'
roadsFileName_Fr = 'Routes_admissibles'
version = self.parameterAsString(
parameters,
self.VERSION,
context
)
# Retrieve the feature source and sink. The 'dest_id' variable is used
# to uniquely identify the feature sink, and must be included in the
# dictionary returned by the processAlgorithm function.
#----------------------------------------
sdcs = self.parameterAsSource(
parameters,
self.SDCS,
context
)
sdccommid = self.parameterAsString(
parameters,
self.SDCCOMMID,
context
)
sdcnameen = self.parameterAsString(
parameters,
self.SDCNAMEEN,
context
)
sdcnamefr = self.parameterAsString(
parameters,
self.SDCNAMEFR,
context
)
sdclat = self.parameterAsString(
parameters,
self.SDCLAT,
context
)
sdclon = self.parameterAsString(
parameters,
self.SDCLON,
context
)
sdcprovterr = self.parameterAsString(
parameters,
self.SDCPROVTERR,
context
)
#-----------------------------------------------------------------------
etcs = self.parameterAsSource(
parameters,
self.ETCS,
context
)
etccommid = self.parameterAsString(
parameters,
self.ETCCOMMID,
context
)
etcnameen = self.parameterAsString(
parameters,
self.ETCNAMEEN,
context
)
etcnamefr = self.parameterAsString(
parameters,
self.ETCNAMEFR,
context
)
etcprovterr = self.parameterAsString(
parameters,
self.ETCPROVTERR,
context
)
etcsatdep = self.parameterAsString(
parameters,
self.ETCSATDEP,
context
)
etclat = self.parameterAsString(
parameters,
self.ETCLAT,
context
)
etclon = self.parameterAsString(
parameters,
self.ETCLON,
context
)
#-----------------------------------------------------------------------
roads = self.parameterAsSource(
parameters,
self.ROADS,
context
)
roadid = self.parameterAsString(
parameters,
self.ROADID,
context
)
roadname = self.parameterAsString(
parameters,
self.ROADNAME,
context
)
roadrank = self.parameterAsString(
parameters,
self.ROADRANK,
context
)
roadclass = self.parameterAsString(
parameters,
self.ROADCLASS,
context
)
roadlengthkm = self.parameterAsString(
parameters,
self.ROADLENGTHKM,
context
)
#-----------------------------------------------------------------------
inputSAVEPATH = self.parameterAsString(
parameters,
self.INPUTSAVEPATH,
context
)
# Check to see if format specific directories exist. If not, create
if not (os.path.exists(inputSAVEPATH+"/EN/mapinfo")):
os.makedirs(inputSAVEPATH+"/EN/mapinfo")
if not (os.path.exists(inputSAVEPATH+"/EN/esri")):
os.makedirs(inputSAVEPATH+"/EN/esri")
if not (os.path.exists(inputSAVEPATH+"/FR/mapinfo")):
os.makedirs(inputSAVEPATH+"/FR/mapinfo")
if not (os.path.exists(inputSAVEPATH+"/FR/esri")):
os.makedirs(inputSAVEPATH+"/FR/esri")
if not (os.path.exists(inputSAVEPATH+"/EN/google")):
os.makedirs(inputSAVEPATH+"/EN/google")
if not (os.path.exists(inputSAVEPATH+"/FR/google")):
os.makedirs(inputSAVEPATH+"/FR/google")
# If source was not found, throw an exception to indicate that the algorithm
# encountered a fatal error. The exception text can be any string, but in this
# case we use the pre-built invalidSourceError method to return a standard
# helper text for when a source cannot be evaluated
if sdcs is None or etcs is None or roads is None:
raise QgsProcessingException(self.invalidSourceError(parameters, self.SDCS))
# Field types: applicable only to refactor fields
# 1 Boolean
# 2 Integer
# 4 Integer 64
# 6 Double
# 10 String
# 14 Date
# 16 DateTime
# REFACTOR Input files with standardized field names
print("***RUNNING REFACTOR***")
# ENGLISH SHORT
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
refsdc=processing.run("native:refactorfields", {
'INPUT':sdcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': sdccommid,'length': 60,'name': 'CommID','precision': 0,'type': 4},
{'expression': sdcnameen,'length': 60,'name': 'Name_en','precision': 0,'type': 10},
{'expression': sdcnamefr,'length': 60,'name': 'Name_fr','precision': 0,'type': 10},
{'expression': sdcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': sdclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': sdclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/sdc_en_short_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refetc=processing.run("native:refactorfields", {
'INPUT':etcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': etccommid,'length': 60,'name': 'CommID','precision': 0,'type': 4},
{'expression': etcnameen,'length': 60,'name': 'Name_en','precision': 0,'type': 10},
{'expression': etcnamefr,'length': 60,'name': 'Name_fr','precision': 0,'type': 10},
{'expression': etcsatdep,'length': 50,'name': 'SATDEP', 'precision':0, 'type': 10},
{'expression': etcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': etclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': etclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/etc_en_short_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refroads=processing.run("native:refactorfields",
{'INPUT':roads.sourceName(),
'FIELDS_MAPPING':[
{'expression': roadid, 'length': 60,'name': 'Road_ID', 'precision': 0,'type': 4},
{'expression': roadname, 'length': 60,'name': 'Road_Name', 'precision': 0,'type': 10},
{'expression': roadrank, 'length': 5, 'name': 'Rank', 'precision': 0,'type': 4},
{'expression': roadclass, 'length': 5, 'name': 'Class', 'precision': 0,'type': 4},
{'expression': roadlengthkm,'length': 15,'name': 'Length_km','precision': 2,'type': 6}],
'OUTPUT':inputSAVEPATH+'/roads_en_short_'+version+'.tab'},
is_child_algorithm=True,
context=context)
# ENGLISH LONG
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
refsdc_long=processing.run("native:refactorfields", {
'INPUT':sdcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': sdccommid,'length': 60,'name': 'Eligible_Community_Number','precision': 0,'type': 4},
{'expression': sdcnameen,'length': 60,'name': 'Eligible_Community_Name_en','precision': 0,'type': 10},
{'expression': sdcnamefr,'length': 60,'name': 'Eligible_Community_Name_fr','precision': 0,'type': 10},
{'expression': sdcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': sdclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': sdclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/sdcs_en_long_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refetc_long=processing.run("native:refactorfields", {
'INPUT':etcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': etccommid,'length': 60,'name': 'Eligible_Community_Number','precision': 0,'type': 4},
{'expression': etcnameen,'length': 60,'name': 'Eligible_Community_Name_en','precision': 0,'type': 10},
{'expression': etcnamefr,'length': 60,'name': 'Eligible_Community_Name_fr','precision': 0,'type': 10},
{'expression': etcsatdep,'length': 50,'name': 'SATDEP', 'precision':0, 'type': 10},
{'expression': etcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': etclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': etclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/etcs_en_long_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refroads_long=processing.run("native:refactorfields",
{'INPUT':roads.sourceName(),
'FIELDS_MAPPING':[
{'expression': roadid, 'length': 60,'name': 'Eligible_Road_ID', 'precision': 0,'type': 4},
{'expression': roadname, 'length': 60,'name': 'Road_Name', 'precision': 0,'type': 10},
{'expression': roadrank, 'length': 5, 'name': 'Rank', 'precision': 0,'type': 4},
{'expression': roadclass, 'length': 5, 'name': 'Class', 'precision': 0,'type': 4},
{'expression': roadlengthkm,'length': 15,'name': 'Length_km','precision': 2,'type': 6}],
'OUTPUT':inputSAVEPATH+'/roads_en_long_'+version+'.tab'},
is_child_algorithm=True,
context=context)
# FRENCH SHORT
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
refsdc_fr=processing.run("native:refactorfields", {
'INPUT':sdcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': sdccommid,'length': 60,'name': 'IDCollect','precision': 0,'type': 4},
{'expression': sdcnameen,'length': 60,'name': 'Nom_en','precision': 0,'type': 10},
{'expression': sdcnamefr,'length': 60,'name': 'Nom_fr','precision': 0,'type': 10},
{'expression': sdcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': sdclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': sdclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/sdcs_fr_short_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refetc_fr=processing.run("native:refactorfields", {
'INPUT':etcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': etccommid,'length': 60,'name': 'IDCollect','precision': 0,'type': 4},
{'expression': etcnameen,'length': 60,'name': 'Nom_en','precision': 0,'type': 10},
{'expression': etcnamefr,'length': 60,'name': 'Nom_fr','precision': 0,'type': 10},
{'expression': etcsatdep,'length': 50,'name': 'SATDEP', 'precision':0, 'type': 10},
{'expression': etcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': etclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': etclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/etcs_fr_short_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refroads_fr=processing.run("native:refactorfields",
{'INPUT':roads.sourceName(),
'FIELDS_MAPPING':[
{'expression': roadid, 'length': 60,'name': 'ID_Route', 'precision': 0,'type': 4},
{'expression': roadname, 'length': 60,'name': 'Nom_Route', 'precision': 0,'type': 10},
{'expression': roadrank, 'length': 5, 'name': 'Rang', 'precision': 0,'type': 4},
{'expression': roadclass, 'length': 5, 'name': 'Classe', 'precision': 0,'type': 4},
{'expression': roadlengthkm,'length': 15,'name': 'Longueur_k','precision': 2,'type': 6}],
'OUTPUT':inputSAVEPATH+'/roads_fr_short_'+version+'.tab'},
is_child_algorithm=True,
context=context)
# FRENCH LONG
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
refsdc_fr_long=processing.run("native:refactorfields", {
'INPUT':sdcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': sdccommid,'length': 60,'name': 'Num_de_la_collectivité_admiss','precision': 0,'type': 4},
{'expression': sdcnameen,'length': 60,'name': 'Nom_de_la_collectivité_en','precision': 0,'type': 10},
{'expression': sdcnamefr,'length': 60,'name': 'Nom_de_la_collectivité_fr','precision': 0,'type': 10},
{'expression': sdcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': sdclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': sdclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/sdcs_fr_long_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refetc_fr_long=processing.run("native:refactorfields", {
'INPUT':etcs.sourceName(),
'FIELDS_MAPPING':[
{'expression': etccommid,'length': 60,'name': 'Num_de_la_collectivité_admiss','precision': 0,'type': 4},
{'expression': etcnameen,'length': 60,'name': 'Nom_de_la_collectivité_en','precision': 0,'type': 10},
{'expression': etcnamefr,'length': 60,'name': 'Nom_de_la_collectivité_fr','precision': 0,'type': 10},
{'expression': etcsatdep,'length': 50,'name': 'SATDEP', 'precision':0, 'type': 10},
{'expression': etcprovterr,'length': 5,'name': 'ProvTerr','precision': 0,'type': 10},
{'expression': etclat,'length': 15,'name': 'Latitude','precision': 6,'type': 6},
{'expression': etclon,'length': 15,'name': 'Longitude','precision': 6,'type': 6}],
'OUTPUT':inputSAVEPATH+'/etcs_fr_long_'+version+'.tab'},
is_child_algorithm=True,
context=context)
refroads_fr_long=processing.run("native:refactorfields",
{'INPUT':roads.sourceName(),
'FIELDS_MAPPING':[
{'expression': roadid, 'length': 60,'name': 'ID_de_la_route_admissible', 'precision': 0,'type': 4},
{'expression': roadname, 'length': 60,'name': 'Nom_de_la_route', 'precision': 0,'type': 10},
{'expression': roadrank, 'length': 5, 'name': 'Rang', 'precision': 0,'type': 4},
{'expression': roadclass, 'length': 5, 'name': 'Classe', 'precision': 0,'type': 4},
{'expression': roadlengthkm,'length': 15,'name': 'Longueur_km','precision': 2,'type': 6}],
'OUTPUT':inputSAVEPATH+'/roads_fr_long_'+version+'.tab'},
is_child_algorithm=True,
context=context)
# Create post-processing objects for renaming layers in layers pane
global sdcrenamer
global etcrenamer
global roadsrenamer
global sdcrenamer_fr
global etcrenamer_fr
global roadsrenamer_fr
sdcrenamer = Renamer(sdcFileName+'_'+version)
etcrenamer = Renamer(etcFileName+'_'+version)
roadsrenamer = Renamer(roadsFileName+'_'+version)
sdcrenamer_fr = Renamer(sdcFileName_Fr+'_'+version)
etcrenamer_fr = Renamer(etcFileName_Fr+'_'+version)
roadsrenamer_fr = Renamer(roadsFileName_Fr+'_'+version)
#context.layerToLoadOnCompletionDetails(refsdc['OUTPUT']).setPostProcessor(sdcrenamer)
#context.layerToLoadOnCompletionDetails(refetc['OUTPUT']).setPostProcessor(etcrenamer)
#context.layerToLoadOnCompletionDetails(refroads['OUTPUT']).setPostProcessor(roadsrenamer)
#context layerToLoadOnCompletionDetails(refsdc_fr['OUTPUT']).setPostProcessor(sdcrenamer_fr)
#context.layerToLoadOnCompletionDetails(refetc_fr['OUTPUT']).setPostProcessor(etcrenamer_fr)
#context.layerToLoadOnCompletionDetails(refroads_fr['OUTPUT']).setPostProcessor(roadsrenamer_fr)
print("***RUNNING SAVE XLSX***")
# Save refactored layers to .shp / .tab and / .xlsx formats
processing.run("native:savefeatures",
{'INPUT': refsdc['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/Eligible_Satellite_Dependent_Communities_'+version+'.xlsx',
'LAYER_NAME':'Eligible_SDCs',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refetc['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/Eligible_Transport_Communities_'+version+'.xlsx',
'LAYER_NAME':'Eligible_Communities',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refroads['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/Eligible_Roads_'+version+'.xlsx',
'LAYER_NAME':'Eligible_Roads',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
#-----------------------------------------------------------------------
# MAPINFO FORMATS
processing.run("native:savefeatures",
{'INPUT': refsdc_long['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/EN/mapinfo/Eligible_Satellite_Dependent_Communities_'+version+'.tab',
'LAYER_NAME':'Eligible_SDCs',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refetc_long['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/EN/mapinfo/Eligible_Transport_Communities_'+version+'.tab',
'LAYER_NAME':'Eligible_Communities',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refroads_long['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/EN/mapinfo/Eligible_Roads_'+version+'.tab',
'LAYER_NAME':'Eligible_Roads',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refsdc_fr_long['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/FR/mapinfo/Collectivités_dépendantes_dun_satellite_admissibles_'+version+'.tab',
'LAYER_NAME':'Collectivités_dépendantes_dun_satellite_admissibles',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refetc_fr_long['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/FR/mapinfo/Collectivités_admissibles_sans_transport_à_grande_capacité_'+version+'.tab',
'LAYER_NAME':'Collectivités_admissibles_sans_transport_à_grande_capacité',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refroads_fr_long['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/FR/mapinfo/Routes_admissibles_'+version+'.tab',
'LAYER_NAME':'Routes_admissibles',
'DATASOURCE_OPTIONS':'overwrite',
'LAYER_OPTIONS':''})
#-----------------------------------------------------------------------
# SHAPEFILES
processing.run("native:savefeatures",
{'INPUT': refsdc['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/EN/esri/Eligible_Satellite_Dependent_Communties_'+version+'.gpkg',
'LAYER_NAME':'Eligible_Satellite_Dependent_Communities',
'DATASOURCE_OPTIONS':'',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refetc['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/EN/esri/Eligible_Transport_Communities_'+version+'.gpkg',
'LAYER_NAME':'Eligible_Transport_Communities',
'DATASOURCE_OPTIONS':'',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refroads['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/EN/esri/Eligible_Roads_'+version+'.shp',
'LAYER_NAME':'Eligible_Roads',
'DATASOURCE_OPTIONS':'',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refsdc_fr['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/FR/esri/Collectivités_dépendantes_dun_satellite_admissibles_'+version+'.shp',
'LAYER_NAME':'Collectivités_dépendantes_dun_satellite_admissibles',
'DATASOURCE_OPTIONS':'',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refetc_fr['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/FR/esri/Collectivités_admissibles_sans_transport_à_grande_capacité_'+version+'.shp',
'LAYER_NAME':'Collectivités_admissibles_sans_transport_à_grande_capacité',
'DATASOURCE_OPTIONS':'',
'LAYER_OPTIONS':''})
processing.run("native:savefeatures",
{'INPUT': refroads_fr['OUTPUT'],
'OUTPUT':inputSAVEPATH+'/FR/esri/Routes_admissibles_'+version+'.shp',
'LAYER_NAME':'Routes_admissibles',
'DATASOURCE_OPTIONS':'',
'LAYER_OPTIONS':''})
#return {'REFSDC': refsdc['OUTPUT'], 'REFETC': refetc['OUTPUT'], 'REFROADS': refroads['OUTPUT']}
return {}
class Renamer (QgsProcessingLayerPostProcessorInterface):
def __init__(self, layer_name):
self.name = layer_name
super().__init__()
def postProcessLayer(self, layer, context, feedback):
layer.setName(self.name)