Convert Json input to CSV output file using powerShell - powershell

I have a json file as given below.
{
"count": 100,
"value": [
{
"id": 264871,
"release": {
"id": 36803,
"name": "Test_1020_SP_1",
"url": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/releases/36803",
"artifacts": [
{
"sourceId": "6d203219-63b3-4b1c-b4fe-fa9172d74fb1/efa62ff5-7dc3-4b5a-b7e1-29a319550b28:12e54cbd-881f-43de-8442-4454ffba61fb",
"type": "PackageManagement",
"alias": "_Project.scripts.release",
"definitionReference": {
"definition": {
"id": "12e54cbd-881f-43de-8442-4454ffba61fb",
"name": "Project.scripts.release"
},
"feed": {
"id": "6d203219-63b3-4b1c-b4fe-fa9172d74fb1/efa62ff5-7dc3-4b5a-b7e1-29a319550b28",
"name": "Project_release_scripts"
},
"files": {
"id": "**",
"name": "**"
},
"packageType": {
"id": "nuget",
"name": "NuGet"
},
"skipextract": {
"id": "",
"name": ""
},
"version": {
"id": "2022.10.11.1",
"name": "2022.10.11.1"
},
"view": {
"id": "",
"name": ""
}
},
"isPrimary": true,
"isRetained": false
}
],
"webAccessUri": "https://dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_release?releaseId=36803\u0026_a=release-summary",
"_links": {
"self": {
"href": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/releases/36803"
},
"web": {
"href": "https://dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_release?releaseId=36803\u0026_a=release-summary"
}
}
},
"releaseDefinition": {
"id": 6777,
"name": "Test_Deploy",
"path": "\\Test",
"projectReference": {
"id": "6d203219-63b3-4b1c-b4fe-fa9172d74fb1",
"name": null
},
"url": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/definitions/6777",
"_links": {
"self": {
"href": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/definitions/6777"
},
"web": {
"href": "https://dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_release?definitionId=6777"
}
}
}
},
{
"id": 264870,
"release": {
"id": 36800,
"name": "Test_2698_SP_1",
"url": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/releases/36800",
"artifacts": [
{
"sourceId": "6d203219-63b3-4b1c-b4fe-fa9172d74fb1/efa62ff5-7dc3-4b5a-b7e1-29a319550b28:12e54cbd-881f-43de-8442-4454ffba61fb",
"type": "PackageManagement",
"alias": "_Project.scripts.release",
"definitionReference": {
"definition": {
"id": "12e54cbd-881f-43de-8442-4454ffba61fb",
"name": "Project.scripts.release"
},
"feed": {
"id": "6d203219-63b3-4b1c-b4fe-fa9172d74fb1/efa62ff5-7dc3-4b5a-b7e1-29a319550b28",
"name": "Project_release_scripts"
},
"files": {
"id": "**",
"name": "**"
},
"packageType": {
"id": "nuget",
"name": "NuGet"
},
"skipextract": {
"id": "",
"name": ""
},
"version": {
"id": "2022.10.11.1",
"name": "2022.10.11.1"
},
"view": {
"id": "",
"name": ""
}
},
"isPrimary": true,
"isRetained": false
}
],
"webAccessUri": "https://dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_release?releaseId=36800\u0026_a=release-summary",
"_links": {
"self": {
"href": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/releases/36800"
},
"web": {
"href": "https://dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_release?releaseId=36800\u0026_a=release-summary"
}
}
},
"releaseDefinition": {
"id": 6777,
"name": "Test_Deploy",
"path": "\\Test",
"projectReference": {
"id": "6d203219-63b3-4b1c-b4fe-fa9172d74fb1",
"name": null
},
"url": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/definitions/6777",
"_links": {
"self": {
"href": "https://vsrm.dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_apis/Release/definitions/6777"
},
"web": {
"href": "https://dev.azure.com/TestProject/6d203219-63b3-4b1c-b4fe-fa9172d74fb1/_release?definitionId=6777"
}
}
}
}
]
}
I wan to convert the above json file to csv format with heading as given below
I tried using the below command
(Get-Content -Path "C:\Test.json") | ConvertFrom-Json | Select-Object -expand value | ConvertFrom-Csv |Out-File C:\Test.csv.
But the value is not coming in the expected format and it is coming without the column names

This should work, though it will generate a column per property (including inner arrays).
I would not expect this to work if the json had nested arrays.
# function to flatten a PSCustomObject into a hashtable of primitives
# this can probably be cleaned up
function flatten($customObj) {
$flatHashtable = #{}
foreach ($entry in #($customObj.PSObject.Properties)) {
$value = $entry.Value
if ($value -is [Array]) {
$valueArray = $value
$size = $value.Count
$keyTemplate = '/{0}/'
} else {
$valueArray = #($value)
$size = 1
$keyTemplate = '/'
}
for ($i = 0; $i -lt $size; $i++) {
$item = $valueArray[$i]
$keySeparator = $keyTemplate -f $i
if ($item -is [PSObject]) {
$subentries = flatten $item # recursive
foreach ($subentry in $subentries.GetEnumerator()) {
$flattenedKey = $entry.Name + $keySeparator + $subentry.Key
if (!($flatHashtable.ContainsKey($flattenedKey))) {
$flatHashtable.Add($flattenedKey, $subentry.Value)
}
}
} else {
if (!($flatHashtable.ContainsKey($entry.Name))) {
$flatHashtable.Add($entry.Name, $item)
}
}
}
}
return $flatHashtable
}
$jsonText = '<your json here>'
$jsonObj = $jsonText | ConvertFrom-Json
# ignore "count" property
$releases = $jsonObj.value
$rows = #()
$releases | ForEach-Object { $rows += (flatten $_) }
$csvColumns = $rows.Keys | Group-Object | Select-Object -ExpandProperty Name | Sort-Object
$rows | Select-Object -Property $csvColumns | ConvertTo-Csv | Set-Content '<path to csv file>'

Related

How to remove recourses from ARM template using PowerShell?

{
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"Connection1Name": {
"type": "string",
"defaultValue": "ApiConnection1"
},
"Connection2Name": {
"type": "string",
"defaultValue": "ApiConnection2"
}
},
"functions": [],
"variables": {},
"resources": [
{
"name": "logicApp1",
"type": "Microsoft.Logic/workflows",
"apiVersion": "2017-07-01",
"location": "[resourceGroup().location]",
"properties": {
"definition": {
"$schema": "https://schema.management.azure.com/schemas/2016-06-01/Microsoft.Logic.json",
"contentVersion": "1.0.0.0",
"parameters": {
},
"triggers": {
},
"actions": {
},
"outputs": {
}
},
"parameters": {
}
}
},
{
"name": "[parameters('Connection1Name')]",
"type": "Microsoft.Web/connections",
"apiVersion": "2016-06-01",
"location": "[resourceGroup().location]",
"tags": {
},
"properties": {
"displayName": "[parameters('Connection1Name')]",
"parameterValues": {
},
"customParameterValues": {
},
"nonSecretParameterValues": {
},
"api": {
"id": "[concat(subscription().id, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/logicAppConnectorApi1')]"
}
}
},
{
"name": "[parameters('Connection2Name')]",
"type": "Microsoft.Web/connections",
"apiVersion": "2016-06-01",
"location": "[resourceGroup().location]",
"tags": {
},
"properties": {
"displayName": "[parameters('Connection2Name')]",
"parameterValues": {
},
"customParameterValues": {
},
"nonSecretParameterValues": {
},
"api": {
"id": "[concat(subscription().id, '/providers/Microsoft.Web/locations/', resourceGroup().location, '/managedApis/logicAppConnectorApi1')]"
}
}
}
],
"outputs": {}
}
Hi all, I am trying something like
$jsonContentObject = Get-Content 'logicApp.json' -Raw | ConvertFrom-Json
$jsonContentObject.resources = $jsonContentObject.resources | Where-Object {$_.type -notmatch 'Microsoft.Web/connections' }
But after that the brackets [] are removed ("resources": [])
I need to do this at the level of the ARM template before deploying it, it is not necessary to do with PowerShell, but I don’t know any other options
I also cannot delete them via PSobject as it is generated via [parameters ('Connection2Name')]
I found a way to solve this problem with the script below
$jsonContentObject = Get-Content 'logicApp.json' -Raw | ConvertFrom-Json
$resourceSquare = #(
$jsonContentObject.resources | Where-Object { $_.type -notmatch 'Microsoft.Web/connections' }
)
$jsonContentObject.resources = $resourceSquare
$jsonContentObject | ConvertTo-Json -Depth 100 | Set-Content 'logicApp.json'
This site was helpful to me in this question https://www.cloudsma.com/2019/05/building-json-payload-in-powershell/

Powershell append to an existing object breaks the format of json

Trying to append an array to a json object but having formatting issues. after appending the desired output format is not a json or as expected.
i have a json object, then converted a json object to custom nested object then trying to append new objects to array which is where am getting formatting issues.
help on this is really appreciated.
below is code
$json = #'
{
"scope": {
"entities": [],
"matches": [
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": ""
}
],
"tagCombination": "AND"
}
]
}
}
'#
# Convert from JSON to a nested custom object.
$obj = $json | ConvertFrom-Json
# Append new objects to the array.
$obj.scope.matches += [pscustomobject] #{ type = 'null'
managementZoneId = 'null'
mzId = 'null'
tags = '[{ "context": "CONTEXTLESS", "key": "DetectedName", "value": "naae22let1ci1.ktb.kontoorbrands.com" }]'
tagCombination = 'AND' }
# Convert back to JSON.
$obj | ConvertTo-Json -Depth 100"
output is:
{
"scope": {
"entities": [],
"matches": [
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": ""
}
],
"tagCombination": "AND"
}
{
"type": "null",
"managementZoneId": "null",
"mzId": "null",
"tags": "[{ \"context\": \"CONTEXTLESS\", \"key\": \"DetectedName\", \"value\": \"naae22let1ci1.ktb.kontoorbrands.com\" }]",
"tagCombination": "AND"
}
]
}
}
desired output :
$obj | ConvertTo-Json -Depth 100
{
"scope": {
"entities": [
],
"matches": [
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": ""
}
],
"tagCombination": "AND"
},
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": ""
}
],
"tagCombination": "AND"
}
]
}
}
As you are converting the json to an object, you need to add the new items as objects too, not as json strings.
Try
$json = #'
{
"scope": {
"entities": [],
"matches": [
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": ""
}
],
"tagCombination": "AND"
}
]
}
}
'#
# Convert from JSON to a nested custom object.
$obj = $json | ConvertFrom-Json
# Append new objects to the array.
$obj.scope.matches += [PsCustomObject]#{
type = $null
managementZoneId = $null
mzId = $null
# the next element is an array of PsCustomObjects, hence the need for #()
tags = #([PsCustomObject]#{
context = "CONTEXTLESS"
key = "DetectedName"
value = "naae22let1ci1.ktb.kontoorbrands.com"
})
tagCombination = 'AND'
}
# Convert back to JSON.
$obj | ConvertTo-Json -Depth 100
Output:
{
"scope": {
"entities": [
],
"matches": [
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": ""
}
],
"tagCombination": "AND"
},
{
"type": null,
"managementZoneId": null,
"mzId": null,
"tags": [
{
"context": "CONTEXTLESS",
"key": "DetectedName",
"value": "naae22let1ci1.ktb.kontoorbrands.com"
}
],
"tagCombination": "AND"
}
]
}
}
P.S. PowerShell does not produce 'pretty' json. If you need to convert it to properly spaced json, see my function Format-Json

Split big JSON file by type

Consider a big JSON in this format(ex: all.json):
[
{
"Name": "abc",
"Type": "movie"
},
{
"Name": "bcd",
"Type": "series"
},
{
"Name": "asd",
"Type": "movie"
},
{
"Name": "sdf",
"Type": "series"
}
]
I want split this file in two files by type
series.json
[
{
"Name": "bcd",
"Type": "series"
},
{
"Name": "sdf",
"Type": "series"
}
]
movie.json
[
{
"Name": "abc",
"Type": "movie"
},
{
"Name": "asd",
"Type": "movie"
}
]
What is the better approach to do this split using powershell? Someone can help?
try this:
#import data from file
$Array=Get-Content "C:\temp\test.json" | ConvertFrom-Json
#group data by type and export
$Array | group Type | %{
$File="C:\temp\{0}.json" -f $_.Name
$_.Group | ConvertTo-Json | Out-File $File
}

Replace values in Json using powershell

I have a json file in which i would like to change values and save again as a Json:
Values that need to be updated:
domain
repo
[
{
"name": "[concat(parameters('factoryName'), '/LS_New')]",
"type": "Microsoft.DataFactory/factories/linkedServices",
"apiVersion": "2018-06-01",
"properties": {
"description": "Connection",
"annotations": [],
"type": "AzureDatabricks",
"typeProperties": {
"domain": "https://url.net",
"accessToken": {
"type": "AzureKeyVaultSecret",
"store": {
"referenceName": "LS_vault",
"type": "LinkedServiceReference"
},
"secretName": "TOKEN"
},
"newClusterNodeType": "Standard_DS4_v2",
"newClusterNumOfWorker": "2:10",
"newClusterSparkEnvVars": {
"PYSPARK_PYTHON": "/databricks/python3/bin/python3"
},
"newClusterVersion": "7.2.x-scala2.12"
}
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LS_evaKeyVault')]"
]
},
{
"name": "[concat(parameters('factoryName'), '/PIP_Log')]",
"type": "Microsoft.DataFactory/factories/pipelines",
"apiVersion": "2018-06-01",
"properties": {
"description": "Unzip",
"activities": [
{
"name": "Parse",
"description": "This notebook",
"type": "DatabricksNotebook",
"dependsOn": [],
"policy": {
"timeout": "7.00:00:00",
"retry": 0,
"retryIntervalInSeconds": 30,
"secureOutput": false,
"secureInput": false
},
"userProperties": [],
"typeProperties": {
"notebookPath": "/dataPipelines/main_notebook.py",
"baseParameters": {
"businessgroup": {
"value": "#pipeline().parameters.businessgroup",
"type": "Expression"
},
"project": {
"value": "#pipeline().parameters.project",
"type": "Expression"
}
},
"libraries": [
{
"pypi": {
"package": "cytoolz"
}
},
{
"pypi": {
"package": "log",
"repo": "https://b73gxyht"
}
}
]
},
"linkedServiceName": {
"referenceName": "LS_o",
"type": "LinkedServiceReference"
}
}
],
"parameters": {
"businessgroup": {
"type": "string",
"defaultValue": "test"
},
"project": {
"type": "string",
"defaultValue": "log-analytics"
}
},
"annotations": []
},
"dependsOn": [
"[concat(variables('factoryId'), '/linkedServices/LS_o')]"
]
}
]
I tried using regex but i am only able to update 1 value :
<valuesToReplace>
<valueToReplace>
<regExSearch>(\/PIP_Log[\w\W]*?[pP]roperties[\w\W]*?[lL]ibraries[\w\W]*?[pP]ypi[\w\W]*?"repo":\s)"(.*?[^\\])"</regExSearch>
<replaceWith>__PATValue__</replaceWith>
</valueToReplace>
<valueToReplace>
<regExSearch>('\/LS_New[\w\W]*?[pP]roperties[\w\W]*?[tT]ypeProperties[\w\W]*?"domain":\s"(.*?[^\\])")</regExSearch>
<replaceWith>__LSDomainName__</replaceWith>
</valueToReplace>
</valuesToReplace>
Here is the powershell code. The loop goes through all the values that are to be replaced.
I tried using dynamic variable in select-string and looping, but it doesn't seem to work
foreach($valueToReplace in $configFile.valuesToReplace.valueToReplace)
{
$regEx = $valueToReplace.regExSearch
$replaceValue = '"' + $valueToReplace.replaceWith + '"'
$matches = [regex]::Matches($json, $regEx)
$matchExactValueRegex = $matches.Value | Select-String -Pattern """repo\D:\s*(.*)" | % {$_.Matches.Groups[1].Value}
$updateReplaceValue = $matches.Value | Select-String -Pattern "repo\D:\s\D__(.*)__""" | % {$_.Matches.Groups[1].Value}
$updateReplaceValue = """$patValue"""
$json1 = [regex]::Replace($json, $matchExactValueRegex , $updateReplaceValue)
$matchExactValueRegex1 = $matches.Value | Select-String -Pattern """domain\D:\s*(.*)" | % {$_.Matches.Groups[1].Value}
$updateReplaceValue1 = $matches.Value | Select-String -Pattern "domain\D:\s\D__(.*)__""" | % {$_.Matches.Groups[1].Value}
$updateReplaceValue1 = """$domainURL"""
$json = [regex]::Replace($json1, $matchExactValueRegex1 , $updateReplaceValue1)
}
else
{
Write-Warning "Inactive config value"
}
$json | Out-File $armFileWithReplacedValues
Where am i missing??
You should not peek and poke in serialized files (as e.g. Json files) directly. Instead deserialize the file with the ConvertFrom-Json cmdlet, make your changes to the object and serialize it again with the ConvertTo-Json cmdlet:
$Data = ConvertFrom-Json $Json
$Data[0].properties.typeproperties.domain = '_LSDomainName__'
$Data[1].properties.activities.typeproperties.libraries[1].pypi.repo = '__PATValue__'
$Data | ConvertTo-Json -Depth 9 | Out-File $armFileWithReplacedValues

Not able to fetch the individual details from JSON data

"Ns": {
"value": [
{
"Nname": "exa",
"SR": [
{
"name": "port1",
"properties": {
"description": "Allow port1",
"destinationPortRange": "1111",
"priority": 100
}
},
{
"name": "port1_0",
"properties": {
"description": "Allow port1",
"destinationPortRange": "1111",
"priority": 150
}
},
{
"name": "port2",
"properties": {
"description": "Allow 1115",
"destinationPortRange": "1115",
"priority": 100,
}
}
]
}
]
}
Want to assert the details of priority and name but was not able to do it.
Here is what I have implemented:
$Ndetails = templateProperties.parameters.Ns.value.SR
foreach ($Ndata in $Ndetails) {
$Ndata .properties.destinationPortRange |
Should -BeExactly #('1111','1111','1115')
} 
How to resolve the same using Pester in PowerShell?
You don't need to use foreach for this. You can just use Select-Object for this. Assuming your JSON is as #Mark Wragg linked in the comments:
$Json = #'
[{
"Ns": {
"value": [{
"Nname": "exa",
"SR": [{
"name": "port1",
"properties": {
"description": "Allow port1",
"destinationPortRange": "1111",
"priority": 100
}
},
{
"name": "port1_0",
"properties": {
"description": "Allow port1",
"destinationPortRange": "1111",
"priority": 150
}
},
{
"name": "port2",
"properties": {
"description": "Allow 1115",
"destinationPortRange": "1115",
"priority": 100
}
}
]
}]
}
}]
'#
$t = $Json | ConvertFrom-Json
Your test file should look like this:
$result = $t.Ns.value.SR.properties.destinationPortRange
it 'destinationPortRange matches' {
$result | Should -BeExactly #('1111','1111','1115')
}
Explanation
Your use of foreach was incorrect as you compared single element (also notice that I deleted unnecessary space)
$Ndata.properties.destinationPortRange
to the array
| Should -BeExactly #('1111','1111','1115')
What you have to do is to compare array to array as in my example.