Looping Creates to many artefacts

Describe the issue/error/question

Hello, I have a loop which retrives me about 32000 entrys from an api (Which limits me to only 100 entrys per request). My n8n instance always crashes because of that. I try to avoid it by saving the 100 entrys into a database on at a time to retrive later. But the problem is still here. I suspect that each node keeps the data from the run/loop before. Is there a solution to that problem ? Can I dump previose items lists in a node ?

Please share the workflow


(Select the nodes and use the keyboard shortcuts CMD+C/CTRL+C and CMD+V/CTRL+V to copy and paste the workflow respectively)

{
  "meta": {
    "instanceId": Preformatted text"48c4be266ae13f250d89fa22dd1eb53a8a060ac337bece9712d9ff3e61a94df9"
  },
  "nodes": [
    {
      "parameters": {
        "fieldToSplitOut": "items",
        "options": {}
      },
      "id": "e810cfe5-4449-4104-8d7c-1290bcdcba19",
      "name": "Item Lists1",
      "type": "n8n-nodes-base.itemLists",
      "typeVersion": 1,
      "position": [
        380,
        140
      ]
    },
    {
      "parameters": {
        "method": "POST",
        "url": "https://api.respond.io/v2/contact/list",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "sendQuery": true,
        "queryParameters": {
          "parameters": [
            {
              "name": "limit",
              "value": "100"
            }
          ]
        },
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "{\"search\":\"\",\"filter\":{\"$and\":[]},\"timezone\":\"Asia/Kuala_Lumpur\"}",
        "options": {}
      },
      "id": "7bbbf6e0-6f1b-4b05-b9fc-350872efb4ec",
      "name": "HTTP Request",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 3,
      "position": [
        -460,
        520
      ],
      "credentials": {
        "httpHeaderAuth": {
          "id": "3",
          "name": "Respond.io Auth Header"
        }
      }
    },
    {
      "parameters": {
        "method": "POST",
        "url": "={{ $json.pagination.next }}",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "{\"search\":\"\",\"filter\":{\"$and\":[]},\"timezone\":\"Asia/Kuala_Lumpur\"}",
        "options": {}
      },
      "id": "cbe658fe-a69b-41ac-844b-50acf02e1816",
      "name": "HTTP Request1",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 3,
      "position": [
        0,
        640
      ],
      "credentials": {
        "httpHeaderAuth": {
          "id": "3",
          "name": "Respond.io Auth Header"
        }
      }
    },
    {
      "parameters": {
        "conditions": {
          "string": [
            {
              "value1": "={{ $json.pagination.next }}",
              "value2": "={{ null }}"
            }
          ]
        },
        "combineOperation": "any"
      },
      "id": "500572fc-34eb-4969-869d-8214218ad993",
      "name": "IF",
      "type": "n8n-nodes-base.if",
      "typeVersion": 1,
      "position": [
        1220,
        520
      ]
    },
    {
      "parameters": {
        "jsCode": "let results = [],\n  i = 0;\n\ndo {\n  try {\n    results = results.concat($(\"MySQL1\").all(0, i));\n  } catch (error) {\n    return results.map(x=>{\n      return x.json.items});\n  }\n  i++;\n} while (true);\n"
      },
      "id": "150e8590-6161-40e1-a1b6-18641b699c51",
      "name": "Code",
      "type": "n8n-nodes-base.code",
      "typeVersion": 1,
      "position": [
        1660,
        500
      ]
    },
    {
      "parameters": {},
      "id": "aeefad68-2a62-42b8-824a-189976b51f51",
      "name": "Execute Workflow Trigger",
      "type": "n8n-nodes-base.executeWorkflowTrigger",
      "typeVersion": 1,
      "position": [
        -680,
        520
      ]
    },
    {
      "parameters": {},
      "id": "4a995a30-f73e-430c-8fe6-108a89994d74",
      "name": "NoOp",
      "type": "n8n-nodes-base.noOp",
      "position": [
        180,
        520
      ],
      "typeVersion": 1
    },
    {
      "parameters": {
        "conditions": {
          "string": [
            {
              "value1": "={{ $json.pagination.next }}",
              "value2": "https://api.respond.io/v2/contact/list?limit=100&cursorId=78680587"
            }
          ]
        },
        "combineOperation": "any"
      },
      "id": "20d9245e-547a-444f-97e2-8b2c0c6fc0e1",
      "name": "IF1",
      "type": "n8n-nodes-base.if",
      "typeVersion": 1,
      "position": [
        1380,
        720
      ]
    },
    {
      "parameters": {
        "conditions": {
          "string": [
            {
              "value1": "={{ $json.pagination.next }}",
              "operation": "notEqual",
              "value2": "={{ null }}"
            }
          ]
        }
      },
      "id": "056c1de9-134b-4904-bb2f-1db736a074e8",
      "name": "IF2",
      "type": "n8n-nodes-base.if",
      "typeVersion": 1,
      "position": [
        900,
        520
      ]
    },
    {
      "parameters": {
        "operation": "aggregateItems",
        "aggregate": "aggregateAllItemData"
      },
      "id": "cfb0bd6a-a44e-4401-b5c9-ac3c8d4fdc0a",
      "name": "Item Lists",
      "type": "n8n-nodes-base.itemLists",
      "typeVersion": 1,
      "position": [
        1920,
        500
      ]
    },
    {
      "parameters": {
        "jsCode": "// Loop over input items and add a new field\n// called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n  item.json.pagination.next = null;\n}\n\nreturn $input.all();"
      },
      "id": "3f010c51-7276-4af9-bb14-acba7bdf1503",
      "name": "Code1",
      "type": "n8n-nodes-base.code",
      "typeVersion": 1,
      "position": [
        -180,
        440
      ]
    },
    {
      "parameters": {
        "table": {
          "__rl": true,
          "value": "CRM.RespondIO",
          "mode": "name"
        },
        "columns": "id,firstName,lastName,phone,email,language,profilePic,locale,countryCode,status,created_at",
        "options": {
          "ignore": true
        }
      },
      "id": "6859eab1-624b-4cd6-85da-7f7ba80f7f8a",
      "name": "MySQL",
      "type": "n8n-nodes-base.mySql",
      "typeVersion": 1,
      "position": [
        820,
        140
      ],
      "credentials": {
        "mySql": {
          "id": "5",
          "name": "CRM"
        }
      }
    },
    {
      "parameters": {
        "operation": "executeQuery",
        "query": "Select * from RespondIO"
      },
      "id": "f6459205-8fb9-4e9d-afc0-d5cccfc0754b",
      "name": "MySQL1",
      "type": "n8n-nodes-base.mySql",
      "typeVersion": 1,
      "position": [
        1440,
        420
      ],
      "credentials": {
        "mySql": {
          "id": "5",
          "name": "CRM"
        }
      }
    },
    {
      "parameters": {
        "batchSize": 1,
        "options": {}
      },
      "id": "fc01b9e6-ee27-48e4-b64a-060eef257b21",
      "name": "Split In Batches",
      "type": "n8n-nodes-base.splitInBatches",
      "typeVersion": 1,
      "position": [
        600,
        140
      ]
    },
    {
      "parameters": {
        "mode": "runOnceForEachItem",
        "jsCode": "// Add a new field called 'myNewField' to the\n// JSON of the item\nvar mysql = require('mysql');\n\nvar con = mysql.createConnection({\n  host: \"localhost\",\n  user: \"yourusername\",\n  password: \"yourpassword\"\n});\n\ncon.connect(function(err) {\n  if (err) throw err;\n  console.log(\"Connected!\");\n  var names = \"id, firstName, lastName, phone, email, language, profilePic, locale, countryCode, status, custom_fields, tags, assignee, created_at\";\n  var values = \"'\"+item[0].json.id+\"','\"+item[0].json.firstName+\"','\"+item[0].json.lastName+\"','\"+item[0].json.phone+\"','\"+item[0].json.email+\"','\"+item[0].json.language+\"','\"+item[0].json.profilePic+\"','\"+item[0].json.locale+\"','\"+item[0].json.countryCode+\"','\"+item[0].json.status+\"','\"+item[0].json.custom_fields+\"','\"+item[0].json.tags+\"','\"+item[0].json.assignee+\"','\"+item[0].json.created_at+\"'\";\n  var sql = \"Insert into `CRM`.`RespondIO` (\"+names+\") VALUES (\"+values+\")\";\n  con.query(sql, function (err, result) {\n    if (err) throw err;\n    console.log(\"1 record inserted\");\n  });\n});\n\n\nreturn $input.item;"
      },
      "id": "e22b59a5-62ad-4041-bba6-85cf3b012ff8",
      "name": "Code2",
      "type": "n8n-nodes-base.code",
      "typeVersion": 1,
      "position": [
        820,
        0
      ]
    },
    {
      "parameters": {
        "mode": "runOnceForEachItem",
        "jsCode": "return {json:{pagination: {next:item.json.pagination.next}}}"
      },
      "id": "9a808968-2b64-465f-a9de-9b2b926626a0",
      "name": "Code3",
      "type": "n8n-nodes-base.code",
      "typeVersion": 1,
      "position": [
        1600,
        820
      ]
    }
  ],
  "connections": {
    "Item Lists1": {
      "main": [
        [
          {
            "node": "Split In Batches",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "HTTP Request": {
      "main": [
        [
          {
            "node": "HTTP Request1",
            "type": "main",
            "index": 0
          },
          {
            "node": "Code1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "HTTP Request1": {
      "main": [
        [
          {
            "node": "NoOp",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "IF": {
      "main": [
        [
          {
            "node": "MySQL1",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "IF1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Code": {
      "main": [
        [
          {
            "node": "Item Lists",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Execute Workflow Trigger": {
      "main": [
        [
          {
            "node": "HTTP Request",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "NoOp": {
      "main": [
        [
          {
            "node": "IF2",
            "type": "main",
            "index": 0
          },
          {
            "node": "Item Lists1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "IF1": {
      "main": [
        [
          {
            "node": "Code",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Code3",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "IF2": {
      "main": [
        [
          {
            "node": "IF",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Code1": {
      "main": [
        [
          {
            "node": "NoOp",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "MySQL1": {
      "main": [
        [
          {
            "node": "Code",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Split In Batches": {
      "main": [
        [
          {
            "node": "MySQL",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Code3": {
      "main": [
        [
          {
            "node": "HTTP Request1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

Information on your n8n setup

  • **n8n version:**0.217.1
  • **Database you’re using (default: SQLite): MariaDB
  • **Running n8n with the execution process [own(default), main]:own
  • **Running n8n via [Docker, npm, n8n.cloud, desktop app]:Docker

Hi @Nuri, welcome to the community :tada:

I am sorry to hear you’re having trouble.

I suspect that each node keeps the data from the run/loop before.

Yes, your assumption is correct. n8n would keep all data from the current workflow execution in memory. Only once your workflow finishes (or the instance crashes) the memory would become available again.

As a workaround you could consider doing the heavy lifting in a sub-workflow, making sure the sub-workflow only returns a small/empty item back to the main workflow. I’ve posted an example workflow at Channel closed Error - #3 by MutedJam.

Let me know if you have any questions on this!

Hi @MutedJam
Thank you for your quick respons and for the tip. I couldn’t make it work with subworkflows but I made it work thanks to databases. While only keeping one the number for the offset in memory and giving this number to a subworkflow. By keeping your tip in mind I managed to make it work. even do it is now very database heavy and very slow :smiley:

1 Like

This topic was automatically closed 90 days after the last reply. New replies are no longer allowed.