Want to get document data from a particular indexId and need to achieve this dynamically

I have an elastic search node which list all the documents available in all the indexId passed to it, Now I was to return an array, which consists of an object inside the object I want the IndexId and all the document id present in that indexId.

Hey @LENINDALLAS,

Do you have some example data to play with? Don’t forget to remove anything important from it.

{
  "nodes": [
    {
      "parameters": {
        "resource": "index",
        "operation": "getAll",
        "returnAll": true
      },
      "name": "  id index elastic search",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        1000,
        660
      ],
      "credentials": {
        "elasticsearchApi": "id credentials"
      }
    },
    {
      "parameters": {
        "operation": "getAll",
        "indexId": "={{$node[\"  id index elastic search\"].json[\"indexId\"]}}",
        "returnAll": true,
        "options": {}
      },
      "name": "id list all",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        1150,
        660
      ],
      "credentials": {
        "elasticsearchApi": "id credentials"
      }
    },
    {
      "parameters": {
        "functionCode": "const metadata = [];\n\nfor (let item of items){\nmetadata.push({json: {\n _id: item.json._id\n }\n });\n}\n//console.log(items);\n\nreturn metadata;\n"
      },
      "name": "id Function",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1330,
        660
      ]
    },
    {
      "parameters": {
        "resource": "index",
        "operation": "getAll",
        "returnAll": true
      },
      "name": "getall index",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        1070,
        400
      ],
      "credentials": {
        "elasticsearchApi": "id credentials"
      }
    },
    {
      "parameters": {
        "httpMethod": "POST",
        "path": "id",
        "responseMode": "lastNode",
        "responseData": "allEntries",
        "options": {}
      },
      "name": "id webhook",
      "type": "n8n-nodes-base.webhook",
      "typeVersion": 1,
      "position": [
        840,
        500
      ],
      "webhookId": "293b0c21-5ce3-4a76-a40c-fd810f8733ea"
    },
    {
      "parameters": {},
      "name": "Merge",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 1,
      "position": [
        1550,
        530
      ]
    },
    {
      "parameters": {
        "functionCode": "\nlet data = [{json:[]}, {json:[]}];\nlet added = {json: []};\n\nitems.forEach((item) => {\nif(item.json.indexId){\ndata[0].json.push(item);\n} else {\ndata[1].json.push(item);\n}\n})\n\ndata[0].json.forEach((value) => {\ndata[1].json.forEach((id) => { \nadded.json.push({\njson: {\nindex: value.json.indexId,\nid: id.json._id\n}\n})\n})\n})\nconsole.log(added);\n\nreturn added.json;"
      },
      "name": "id split function",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1710,
        530
      ]
    },
    {
      "parameters": {
        "indexId": "={{$node[\"id split function\"].json[\"index\"]}}",
        "documentId": "={{$node[\"id split function\"].json[\"id\"]}}",
        "options": {}
      },
      "name": "each list get elastic",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        1880,
        530
      ],
      "alwaysOutputData": false,
      "executeOnce": false,
      "credentials": {
        "elasticsearchApi": "id credentials"
      }
    }
  ],
  "connections": {
    "  id index elastic search": {
      "main": [
        [
          {
            "node": "id list all",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "id list all": {
      "main": [
        [
          {
            "node": "id Function",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "id Function": {
      "main": [
        [
          {
            "node": "Merge",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "getall index": {
      "main": [
        [
          {
            "node": "Merge",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "id webhook": {
      "main": [
        [
          {
            "node": "  id index elastic search",
            "type": "main",
            "index": 0
          },
          {
            "node": "getall index",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Merge": {
      "main": [
        [
          {
            "node": "id split function",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "id split function": {
      "main": [
        [
          {
            "node": "each list get elastic",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

This is the workflow constructed, Id split function does the work but the next elasticsearch node does not output any data even when the value is hard coded.
Not able to iterate the values

Hi @LENINDALLAS, I was having a look into this just now but wanted to make sure I look at the exact data structure you’re having on your end. Would you be able to share the JSON output (minus anything confidential or private, of course) from your id split function node?

Hi @MutedJam

[

{

"index": ".ent-search-db-lock-20200304",

"id": "canvas-workpad-template:workpad-template-aefa8b2b-24ec-4093-8a59-f2cbc5f7c947"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "canvas-workpad-template:workpad-template-061d7868-2b4e-4dc8-8bf7-3772b52926e5"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "canvas-workpad-template:workpad-template-6181471b-147d-4397-a0d3-1c0f1600fa12"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "canvas-workpad-template:workpad-template-029bdeb3-40a6-4c90-9320-a5566abaf427"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "canvas-workpad-template:workpad-template-890b80e5-a3eb-431d-b8ed-37587ffd32c3"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "ingest-package-policies:a4496a2f-f9d5-4d80-b985-0e76845228de"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "ingest-agent-policies:policy-elastic-agent-on-cloud"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "ingest-agent-policies:0fe01d90-2bdf-11ec-a193-67e6c2df97e5"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "osquery-manager-usage-metric:live_query"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "index-pattern:ff959d40-b880-11e8-a6d9-e546fe2bba5f"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "account_id:61666230d1fec7da7221e911|user_oid:61666230d1fec7da7221e912"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "account_id:61666230d1fec7da7221e911|user_oid:616e8ea4d1fec7e40929b148"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "000000000000000000000001"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "4ebf6c6b21f4-1076901e-07fe-438a-ad26-8b2ed5b84681"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "61666231d1fec7da7221e913"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "616e8ea4d1fec7e40929b149"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "61666230d1fec7da7221e910"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "uFC1KUAbOMxhlUefvjAh33YPt0b7jUl+XeCsdsANzSI="

},

{

"index": ".ent-search-db-lock-20200304",

"id": "y8Taq/C+gmeLUfAtkymyJYeCe7naL/Lt+z9yp7ZUwXs="

},

{

"index": ".ent-search-db-lock-20200304",

"id": "AEzOK1k4WDOzaZWTTwCrUncI++wDJGNEPNImQ4ZbKkw="

},

{

"index": ".ent-search-db-lock-20200304",

"id": "NczwzoT//iZpg8Wju3jOCpu0gCCirgtxWJqgBLx4cMQ="

},

{

"index": ".ent-search-db-lock-20200304",

"id": "uPDngBhYdUUBV83RDtsK59l7c2uBqzs5kGO5gGuiYv8="

},

{

"index": ".ent-search-db-lock-20200304",

"id": "n+oTYZMo1H+qJH6ivxREasWAyLYSlhLnK/CcVPseDQg="

},

{

"index": ".ent-search-db-lock-20200304",

"id": "uBdjl3wB5U-7ae04F21M"

},

{

"index": ".ent-search-db-lock-20200304",

"id": "uRdjl3wB5U-7ae04F21M"

}

]

This is the response data from the id split function, now I want to perform the get method in the next node by passing index Id and document Id.
Thank you

Many thanks for your example! It looks like the indices you are retrieving are system ones (seeing their names start with a dot . as per What's the difference between dot indices and non-dot indices - #2 by dadoonet - Elasticsearch - Discuss the Elastic Stack). Digging a bit more on the Elastic forums suggests the best course of action would be to avoid such indices.

So if you only want to fetch documents from your own indices rather than system ones, these documents would typically have the index in the document itself. For example:

Meaning if you are only looking for such non-system documents, you could use a much simpler logic like the one below to get IDs and indices for these documents:

{
  "nodes": [
    {
      "parameters": {},
      "name": "Start",
      "type": "n8n-nodes-base.start",
      "typeVersion": 1,
      "position": [
        -150,
        100
      ]
    },
    {
      "parameters": {
        "operation": "getAll",
        "indexId": "={{$json[\"indexId\"]}}",
        "limit": 500,
        "options": {}
      },
      "name": "Get all Documents",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        450,
        100
      ],
      "credentials": {
        "elasticsearchApi": {
          "id": "5",
          "name": "Elasticsearch account"
        }
      }
    },
    {
      "parameters": {
        "resource": "index",
        "operation": "getAll",
        "returnAll": true
      },
      "name": "Get all Indices",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        50,
        100
      ],
      "credentials": {
        "elasticsearchApi": {
          "id": "5",
          "name": "Elasticsearch account"
        }
      }
    },
    {
      "parameters": {
        "keepOnlySet": true,
        "values": {
          "string": [
            {
              "name": "id",
              "value": "={{$json[\"_id\"]}}"
            },
            {
              "name": "index",
              "value": "={{$json[\"index\"]}}"
            }
          ]
        },
        "options": {}
      },
      "name": "Keep only id and index",
      "type": "n8n-nodes-base.set",
      "typeVersion": 1,
      "position": [
        650,
        100
      ]
    },
    {
      "parameters": {
        "conditions": {
          "string": [
            {
              "value1": "={{$json[\"indexId\"]}}",
              "operation": "startsWith",
              "value2": "."
            }
          ],
          "boolean": []
        }
      },
      "name": "System Index?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 1,
      "position": [
        250,
        100
      ]
    }
  ],
  "connections": {
    "Start": {
      "main": [
        [
          {
            "node": "Get all Indices",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get all Documents": {
      "main": [
        [
          {
            "node": "Keep only id and index",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get all Indices": {
      "main": [
        [
          {
            "node": "System Index?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "System Index?": {
      "main": [
        null,
        [
          {
            "node": "Get all Documents",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

Would this already work for you?

Thank you so much @MutedJam Your explanation helped the workflow get way simpler and I understood about the concept a little more

@MutedJam The output got at the last set node does not include index id, it returns only the id.
Can you suggest some ways to get index Id

Hi @LENINDALLAS, if the index is also unavailable in your actual documents (rather than the system ones), my suggestion would be to use the Split node to create a separate batch for each index like so:

{
  "nodes": [
    {
      "parameters": {},
      "name": "Start",
      "type": "n8n-nodes-base.start",
      "typeVersion": 1,
      "position": [
        -150,
        100
      ]
    },
    {
      "parameters": {
        "resource": "index",
        "operation": "getAll",
        "returnAll": true
      },
      "name": "Get all Indices",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        50,
        100
      ],
      "credentials": {
        "elasticsearchApi": {
          "id": "5",
          "name": "Elasticsearch account"
        }
      }
    },
    {
      "parameters": {
        "batchSize": 1,
        "options": {}
      },
      "name": "Split",
      "type": "n8n-nodes-base.splitInBatches",
      "typeVersion": 1,
      "position": [
        250,
        100
      ]
    },
    {
      "parameters": {
        "operation": "getAll",
        "indexId": "={{$json[\"indexId\"]}}",
        "limit": 5,
        "options": {}
      },
      "name": "Get All Docs for Index",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        550,
        400
      ],
      "credentials": {
        "elasticsearchApi": {
          "id": "5",
          "name": "Elasticsearch account"
        }
      }
    },
    {
      "parameters": {
        "mode": "multiplex"
      },
      "name": "Merge Index and Doc",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 1,
      "position": [
        700,
        250
      ]
    },
    {
      "parameters": {
        "keepOnlySet": true,
        "values": {
          "string": [
            {
              "name": "indexId",
              "value": "={{$json[\"indexId\"]}}"
            },
            {
              "name": "id",
              "value": "={{$json[\"_id\"]}}"
            }
          ]
        },
        "options": {}
      },
      "name": "Keep Only Index & Doc ID",
      "type": "n8n-nodes-base.set",
      "typeVersion": 1,
      "position": [
        900,
        100
      ]
    },
    {
      "parameters": {},
      "name": "NoOp",
      "type": "n8n-nodes-base.noOp",
      "typeVersion": 1,
      "position": [
        400,
        250
      ]
    }
  ],
  "connections": {
    "Start": {
      "main": [
        [
          {
            "node": "Get all Indices",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get all Indices": {
      "main": [
        [
          {
            "node": "Split",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Split": {
      "main": [
        [
          {
            "node": "NoOp",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get All Docs for Index": {
      "main": [
        [
          {
            "node": "Merge Index and Doc",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "Merge Index and Doc": {
      "main": [
        [
          {
            "node": "Keep Only Index & Doc ID",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "NoOp": {
      "main": [
        [
          {
            "node": "Get All Docs for Index",
            "type": "main",
            "index": 0
          },
          {
            "node": "Merge Index and Doc",
            "type": "main",
            "index": 0
          },
          {
            "node": "Split",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

This will also work for the aforementioned system indices. You would need to flip through each execution to see its data:


Hey @MutedJam the output is having both index Id and id fields, but since the set node gets executed many number of times the response sent for the webhook call consists only the last set node run value (2nd screenshot attached), could you provide a way, that the response data consists of all the data together.

Sorry, I was looking into the Elasticsearch part of your question and forgot about the webhook :see_no_evil::see_no_evil:

This is still possible though. I’d suggest creating two separate workflows in this case to separate fetching indices and delivering the response from fetching each document. You could then use the logic from https://n8n.io/workflows/1160 to merge the data from each execution back together.

Workflow 1 (fetching indices and handling the webhook):

{
  "nodes": [
    {
      "parameters": {},
      "name": "Start",
      "type": "n8n-nodes-base.start",
      "typeVersion": 1,
      "position": [
        -150,
        100
      ]
    },
    {
      "parameters": {
        "resource": "index",
        "operation": "getAll",
        "returnAll": true
      },
      "name": "Get all Indices",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        50,
        100
      ],
      "credentials": {
        "elasticsearchApi": {
          "id": "5",
          "name": "Elasticsearch account"
        }
      }
    },
    {
      "parameters": {
        "batchSize": 1,
        "options": {}
      },
      "name": "Split",
      "type": "n8n-nodes-base.splitInBatches",
      "typeVersion": 1,
      "position": [
        250,
        100
      ]
    },
    {
      "parameters": {
        "workflowId": "2"
      },
      "name": "Fetch Documents",
      "type": "n8n-nodes-base.executeWorkflow",
      "typeVersion": 1,
      "position": [
        450,
        100
      ],
      "alwaysOutputData": true
    },
    {
      "parameters": {
        "functionCode": "const allData = []\n\nlet counter = 0;\ndo {\n  try {\n    const items = $items(\"Fetch Documents\", 0, counter).map(item => item.json).filter(item => item.hasOwnProperty(\"id\"));\n    allData.push.apply(allData, items);\n  } catch (error) {\n    return allData.map(item => {\n      return {\n        json: item\n      }\n    });  \n  }\n\n  counter++;\n} while(true);\n\n\n"
      },
      "name": "Merge Data",
      "type": "n8n-nodes-base.function",
      "position": [
        850,
        100
      ],
      "typeVersion": 1
    },
    {
      "parameters": {
        "conditions": {
          "boolean": [
            {
              "value1": true,
              "value2": "={{$node[\"Split\"].context[\"noItemsLeft\"]}}"
            }
          ]
        }
      },
      "name": "IF",
      "type": "n8n-nodes-base.if",
      "position": [
        650,
        250
      ],
      "typeVersion": 1
    },
    {
      "parameters": {
        "path": "85f95ca5-43fe-4eb1-9f0b-ec489cf0f91a",
        "responseMode": "lastNode",
        "responseData": "allEntries",
        "options": {}
      },
      "name": "Webhook",
      "type": "n8n-nodes-base.webhook",
      "typeVersion": 1,
      "position": [
        -150,
        250
      ],
      "webhookId": "85f95ca5-43fe-4eb1-9f0b-ec489cf0f91a"
    }
  ],
  "connections": {
    "Get all Indices": {
      "main": [
        [
          {
            "node": "Split",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Split": {
      "main": [
        [
          {
            "node": "Fetch Documents",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Fetch Documents": {
      "main": [
        [
          {
            "node": "IF",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "IF": {
      "main": [
        [
          {
            "node": "Merge Data",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Split",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Webhook": {
      "main": [
        [
          {
            "node": "Get all Indices",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

Workflow 2:

{
  "nodes": [
    {
      "parameters": {},
      "name": "Start",
      "type": "n8n-nodes-base.start",
      "typeVersion": 1,
      "position": [
        -350,
        100
      ]
    },
    {
      "parameters": {
        "operation": "getAll",
        "indexId": "={{$json[\"indexId\"]}}",
        "limit": 500,
        "options": {}
      },
      "name": "Get All Docs for Index",
      "type": "n8n-nodes-base.elasticsearch",
      "typeVersion": 1,
      "position": [
        -200,
        240
      ],
      "credentials": {
        "elasticsearchApi": {
          "id": "5",
          "name": "Elasticsearch account"
        }
      }
    },
    {
      "parameters": {
        "mode": "multiplex"
      },
      "name": "Merge Index and Doc",
      "type": "n8n-nodes-base.merge",
      "typeVersion": 1,
      "position": [
        -50,
        100
      ]
    },
    {
      "parameters": {
        "keepOnlySet": true,
        "values": {
          "string": [
            {
              "name": "indexId",
              "value": "={{$json[\"indexId\"]}}"
            },
            {
              "name": "id",
              "value": "={{$json[\"_id\"]}}"
            }
          ]
        },
        "options": {}
      },
      "name": "Keep Only Index & Doc ID",
      "type": "n8n-nodes-base.set",
      "typeVersion": 1,
      "position": [
        150,
        100
      ]
    }
  ],
  "connections": {
    "Start": {
      "main": [
        [
          {
            "node": "Get All Docs for Index",
            "type": "main",
            "index": 0
          },
          {
            "node": "Merge Index and Doc",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Get All Docs for Index": {
      "main": [
        [
          {
            "node": "Merge Index and Doc",
            "type": "main",
            "index": 1
          }
        ]
      ]
    },
    "Merge Index and Doc": {
      "main": [
        [
          {
            "node": "Keep Only Index & Doc ID",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

Note that workflow 1 refers to workflow 2 via an ID in the Execute Workflow node which likely differs on your end. So you’d need to adjust that. When put together this should give you all the indices and documents:

Hope this helps!

Thank you so much @MutedJam your replies helped to get solution.

2 Likes