I have an use case, where user is defined a workflow. As user hits the API, step function gets started. A lambda returns a list of steps based on users & start the workflow e.g.:
step1. it process(extracts) the doc,
step2. inserts some data to
DB
step3. .....
step4. .....
here steps are in map, as these steps are returned from lambda.
Is there any way to get response from each map iteration, so that I can show user at which step the process is , & the output of previous step.
here is my step function design:
test-fetch-input-workflow: returns an array: [step1, step2, ....]
workflow step-lambda: calls some lambda based on step name
Assuming that you are using Standard Workflows, you can get this information using the GetExecutionHistory API Action: https://docs.aws.amazon.com/step-functions/latest/apireference/API_GetExecutionHistory.html
You will want to look for events of type MapIterationSucceeded, grab the previousEventId, then look for the output in the HistoryEvent with that Id.
For example, if you had a state machine like this:
{
"StartAt": "Generate Array for Map",
"States": {
"Generate Array for Map": {
"Type": "Pass",
"Next": "Map",
"Result": [
1,
2,
3,
4,
5
]
},
"Map": {
"Type": "Map",
"ItemProcessor": {
"ProcessorConfig": {
"Mode": "INLINE"
},
"StartAt": "Generate Random Number",
"States": {
"Generate Random Number": {
"Type": "Pass",
"End": true,
"Parameters": {
"Random Number.$": "States.MathRandom(1,1000)"
}
}
}
},
"End": true
}
}
}
Your execution history would look like this:
{
"events": [
{
"timestamp": "2022-12-08T16:44:07.651000-08:00",
"type": "ExecutionStarted",
"id": 1,
"previousEventId": 0,
"executionStartedEventDetails": {
"input": "{\n \"Comment\": \"Insert your JSON here\"\n}",
"inputDetails": {
"truncated": false
},
"roleArn": "arn:aws:iam::000000000000:role/service-role/StepFunctions-MapTest-role-613230dd"
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateEntered",
"id": 2,
"previousEventId": 0,
"stateEnteredEventDetails": {
"name": "Generate Array for Map",
"input": "{\n \"Comment\": \"Insert your JSON here\"\n}",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateExited",
"id": 3,
"previousEventId": 2,
"stateExitedEventDetails": {
"name": "Generate Array for Map",
"output": "[1,2,3,4,5]",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapStateEntered",
"id": 4,
"previousEventId": 3,
"stateEnteredEventDetails": {
"name": "Map",
"input": "[1,2,3,4,5]",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapStateStarted",
"id": 5,
"previousEventId": 4,
"mapStateStartedEventDetails": {
"length": 5
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationStarted",
"id": 6,
"previousEventId": 5,
"mapIterationStartedEventDetails": {
"name": "Map",
"index": 0
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationStarted",
"id": 7,
"previousEventId": 5,
"mapIterationStartedEventDetails": {
"name": "Map",
"index": 1
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationStarted",
"id": 8,
"previousEventId": 5,
"mapIterationStartedEventDetails": {
"name": "Map",
"index": 2
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationStarted",
"id": 9,
"previousEventId": 5,
"mapIterationStartedEventDetails": {
"name": "Map",
"index": 3
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationStarted",
"id": 10,
"previousEventId": 5,
"mapIterationStartedEventDetails": {
"name": "Map",
"index": 4
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateEntered",
"id": 11,
"previousEventId": 6,
"stateEnteredEventDetails": {
"name": "Generate Random Number",
"input": "1",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateExited",
"id": 12,
"previousEventId": 11,
"stateExitedEventDetails": {
"name": "Generate Random Number",
"output": "{\"Random Number\":259}",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationSucceeded",
"id": 13,
"previousEventId": 12,
"mapIterationSucceededEventDetails": {
"name": "Map",
"index": 0
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateEntered",
"id": 14,
"previousEventId": 7,
"stateEnteredEventDetails": {
"name": "Generate Random Number",
"input": "2",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateExited",
"id": 15,
"previousEventId": 14,
"stateExitedEventDetails": {
"name": "Generate Random Number",
"output": "{\"Random Number\":776}",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationSucceeded",
"id": 16,
"previousEventId": 15,
"mapIterationSucceededEventDetails": {
"name": "Map",
"index": 1
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateEntered",
"id": 17,
"previousEventId": 8,
"stateEnteredEventDetails": {
"name": "Generate Random Number",
"input": "3",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateExited",
"id": 18,
"previousEventId": 17,
"stateExitedEventDetails": {
"name": "Generate Random Number",
"output": "{\"Random Number\":743}",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationSucceeded",
"id": 19,
"previousEventId": 18,
"mapIterationSucceededEventDetails": {
"name": "Map",
"index": 2
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateEntered",
"id": 20,
"previousEventId": 9,
"stateEnteredEventDetails": {
"name": "Generate Random Number",
"input": "4",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateExited",
"id": 21,
"previousEventId": 20,
"stateExitedEventDetails": {
"name": "Generate Random Number",
"output": "{\"Random Number\":790}",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationSucceeded",
"id": 22,
"previousEventId": 21,
"mapIterationSucceededEventDetails": {
"name": "Map",
"index": 3
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateEntered",
"id": 23,
"previousEventId": 10,
"stateEnteredEventDetails": {
"name": "Generate Random Number",
"input": "5",
"inputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "PassStateExited",
"id": 24,
"previousEventId": 23,
"stateExitedEventDetails": {
"name": "Generate Random Number",
"output": "{\"Random Number\":632}",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapIterationSucceeded",
"id": 25,
"previousEventId": 24,
"mapIterationSucceededEventDetails": {
"name": "Map",
"index": 4
}
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapStateSucceeded",
"id": 26,
"previousEventId": 25
},
{
"timestamp": "2022-12-08T16:44:07.698000-08:00",
"type": "MapStateExited",
"id": 27,
"previousEventId": 25,
"stateExitedEventDetails": {
"name": "Map",
"output": "[{\"Random Number\":259},{\"Random Number\":776},{\"Random Number\":743},{\"Random Number\":790},{\"Random Number\":632}]",
"outputDetails": {
"truncated": false
}
}
},
{
"timestamp": "2022-12-08T16:44:07.745000-08:00",
"type": "ExecutionSucceeded",
"id": 28,
"previousEventId": 27,
"executionSucceededEventDetails": {
"output": "[{\"Random Number\":259},{\"Random Number\":776},{\"Random Number\":743},{\"Random Number\":790},{\"Random Number\":632}]",
"outputDetails": {
"truncated": false
}
}
}
]
}
I try to filter some results data from mongodb with mongoose in javascript.
This is my json structure:
{
"name": "john",
"firstname": "doe",
"yearold": 22,
"recipes": [
{
"title": "cheesecake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": 6,
"unit": "piece"
},
{
"name": "oil",
"label": "Specific oil",
"unit": "oz",
"value": 0.2
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 450,
"unit": "gr"
}
]
},
{
"title": "cake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": 6,
"unit": "piece"
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 500,
"unit": "gr"
},
]
}
]
}
In some case i need to return json data with hiding some values. For example I have a list that specifies all the values to hide
hidekeys=["egg"];
and i would like to get this:
{
"name": "john",
"firstname": "doe",
"yearold": 22,
"recipes": [
{
"title": "cheesecake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": #######,
"unit": "piece"
},
{
"name": "oil",
"label": "Specific oil",
"unit": "oz",
"value": 0.2
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 450,
"unit": "gr"
}
]
},
{
"title": "cake",
"data": [
{
"name": "egg",
"label": "Eggs for",
"value": #######,
"unit": "piece"
},
{
"name": "flour",
"label": "Wholemel flour",
"value": 500,
"unit": "gr"
},
]
}
]
}
For each recipe i need to hide ingredient value if it is specified in hidekeys.
I tried something with $project and $cond but it doesnt works
Here's a quick way of how to achieve this using $map
const hidekeys = ["egg"];
db.collection.aggregate([
{
$addFields: {
recipes: {
$map: {
input: "$recipes",
as: "recipe",
in: {
$mergeObjects: [
"$$recipe",
{
data: {
$map: {
input: "$$recipe.data",
as: "datum",
in: {
"$mergeObjects": [
"$$datum",
{
$cond: [
{
"$setIsSubset": [
[
"$$datum.name"
],
hidekeys
]
},
{
value: "#####"
},
{
value: "$$datum.value"
}
]
}
]
}
}
}
}
]
}
}
}
}
}
])
Mongo Playground
Having the following record
{
"name": "
Festões Plástico, 12mt x 17cm - Festas Populares",
"categories": [
"Festas",
"Casamentos",
"Decorações"
],
"hierarchicalCategories": {
"lvl0": "Festas",
"lvl1": "Festas > Casamentos",
"lvl2": "Festas > Casamentos > Decorações"
},
"description": "",
"brand": "Misterius",
"price": 14.94,
"stock": "Disponível",
"prices": [
{
"value": 12,
"type": "specificValue",
"family": "fatos",
"subfamily": "example"
},
{
"value": 13,
"type": "specificValue13",
"family": "fatos13",
"subfamily": "example13"
},
{
"value": 14,
"type": "specificValue14",
"family": "fatos14",
"subfamily": "example14"
},
{
"value": 15,
"type": "specificValue15",
"family": "fatos15",
"subfamily": "example15"
},
{
"value": 16,
"type": "specificValue16",
"family": "fatos16",
"subfamily": "example16"
}
],
"color": [
{
"name": "Amarelo",
"label": "Amarelo,#FFFF00",
"hexa": "#FFFF00"
},
{
"name": "Azul",
"label": "Azul,#0000FF",
"hexa": "#0000FF"
},
{
"name": "Branco",
"label": "Branco,#FFFFFF",
"hexa": "#FFFFFF"
},
{
"name": "Laranja",
"label": "Laranja,#FFA500",
"hexa": "#FFA500"
},
{
"name": "Verde Escuro",
"label": "Verde Escuro,#006400",
"hexa": "#006400"
},
{
"name": "Vermelho",
"label": "Vermelho,#FF0000",
"hexa": "#FF0000"
}
],
"specialcategorie": "",
"reference": "3546",
"rating": 0,
"free_shipping": false,
"popularity": 0,
"objectID": "30"
}
Now by searching for "Festas Populares" will return the record and its attributes, is it possible to also filter for one attribute array as "prices" to only return one json. for example "prices.type"="specificValue14" and "family"="fatos14" and "family"="fatos" and "subfamily"="example"
{
“value”: 14,
“type”: “specificValue14”,
“family”: “fatos14”,
“subfamily”: “example14”
}
the record return would be:
{
"name": "
Festões Plástico, 12mt x 17cm - Festas Populares",
"categories": [
"Festas",
"Casamentos",
"Decorações"
],
"hierarchicalCategories": {
"lvl0": "Festas",
"lvl1": "Festas > Casamentos",
"lvl2": "Festas > Casamentos > Decorações"
},
"description": "",
"brand": "Misterius",
"price": 14.94,
"stock": "Disponível",
"prices": [
{
"value": 14,
"type": "specificValue14",
"family": "fatos14",
"subfamily": "example14"
}
],
"color": [
{
"name": "Amarelo",
"label": "Amarelo,#FFFF00",
"hexa": "#FFFF00"
},
{
"name": "Azul",
"label": "Azul,#0000FF",
"hexa": "#0000FF"
},
{
"name": "Branco",
"label": "Branco,#FFFFFF",
"hexa": "#FFFFFF"
},
{
"name": "Laranja",
"label": "Laranja,#FFA500",
"hexa": "#FFA500"
},
{
"name": "Verde Escuro",
"label": "Verde Escuro,#006400",
"hexa": "#006400"
},
{
"name": "Vermelho",
"label": "Vermelho,#FF0000",
"hexa": "#FF0000"
}
],
"specialcategorie": "",
"reference": "3546",
"rating": 0,
"free_shipping": false,
"popularity": 0,
"objectID": "30"
}
for some context a product can have multiple prices associated, for a specific user, or one day there is campaign giving discount, etc so for that cases want to filter price associated to the product/record.
No, this is not possible with Algolia. Records are always returned with the attributes specified inside attributesToRetrieve. These attributes are returned in full.