Describe the issue/error/question
Hello, I have a loop which retrives me about 32000 entrys from an api (Which limits me to only 100 entrys per request). My n8n instance always crashes because of that. I try to avoid it by saving the 100 entrys into a database on at a time to retrive later. But the problem is still here. I suspect that each node keeps the data from the run/loop before. Is there a solution to that problem ? Can I dump previose items lists in a node ?
Please share the workflow
(Select the nodes and use the keyboard shortcuts CMD+C/CTRL+C and CMD+V/CTRL+V to copy and paste the workflow respectively)
{
"meta": {
"instanceId": Preformatted text"48c4be266ae13f250d89fa22dd1eb53a8a060ac337bece9712d9ff3e61a94df9"
},
"nodes": [
{
"parameters": {
"fieldToSplitOut": "items",
"options": {}
},
"id": "e810cfe5-4449-4104-8d7c-1290bcdcba19",
"name": "Item Lists1",
"type": "n8n-nodes-base.itemLists",
"typeVersion": 1,
"position": [
380,
140
]
},
{
"parameters": {
"method": "POST",
"url": "https://api.respond.io/v2/contact/list",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendQuery": true,
"queryParameters": {
"parameters": [
{
"name": "limit",
"value": "100"
}
]
},
"sendBody": true,
"specifyBody": "json",
"jsonBody": "{\"search\":\"\",\"filter\":{\"$and\":[]},\"timezone\":\"Asia/Kuala_Lumpur\"}",
"options": {}
},
"id": "7bbbf6e0-6f1b-4b05-b9fc-350872efb4ec",
"name": "HTTP Request",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 3,
"position": [
-460,
520
],
"credentials": {
"httpHeaderAuth": {
"id": "3",
"name": "Respond.io Auth Header"
}
}
},
{
"parameters": {
"method": "POST",
"url": "={{ $json.pagination.next }}",
"authentication": "genericCredentialType",
"genericAuthType": "httpHeaderAuth",
"sendBody": true,
"specifyBody": "json",
"jsonBody": "{\"search\":\"\",\"filter\":{\"$and\":[]},\"timezone\":\"Asia/Kuala_Lumpur\"}",
"options": {}
},
"id": "cbe658fe-a69b-41ac-844b-50acf02e1816",
"name": "HTTP Request1",
"type": "n8n-nodes-base.httpRequest",
"typeVersion": 3,
"position": [
0,
640
],
"credentials": {
"httpHeaderAuth": {
"id": "3",
"name": "Respond.io Auth Header"
}
}
},
{
"parameters": {
"conditions": {
"string": [
{
"value1": "={{ $json.pagination.next }}",
"value2": "={{ null }}"
}
]
},
"combineOperation": "any"
},
"id": "500572fc-34eb-4969-869d-8214218ad993",
"name": "IF",
"type": "n8n-nodes-base.if",
"typeVersion": 1,
"position": [
1220,
520
]
},
{
"parameters": {
"jsCode": "let results = [],\n i = 0;\n\ndo {\n try {\n results = results.concat($(\"MySQL1\").all(0, i));\n } catch (error) {\n return results.map(x=>{\n return x.json.items});\n }\n i++;\n} while (true);\n"
},
"id": "150e8590-6161-40e1-a1b6-18641b699c51",
"name": "Code",
"type": "n8n-nodes-base.code",
"typeVersion": 1,
"position": [
1660,
500
]
},
{
"parameters": {},
"id": "aeefad68-2a62-42b8-824a-189976b51f51",
"name": "Execute Workflow Trigger",
"type": "n8n-nodes-base.executeWorkflowTrigger",
"typeVersion": 1,
"position": [
-680,
520
]
},
{
"parameters": {},
"id": "4a995a30-f73e-430c-8fe6-108a89994d74",
"name": "NoOp",
"type": "n8n-nodes-base.noOp",
"position": [
180,
520
],
"typeVersion": 1
},
{
"parameters": {
"conditions": {
"string": [
{
"value1": "={{ $json.pagination.next }}",
"value2": "https://api.respond.io/v2/contact/list?limit=100&cursorId=78680587"
}
]
},
"combineOperation": "any"
},
"id": "20d9245e-547a-444f-97e2-8b2c0c6fc0e1",
"name": "IF1",
"type": "n8n-nodes-base.if",
"typeVersion": 1,
"position": [
1380,
720
]
},
{
"parameters": {
"conditions": {
"string": [
{
"value1": "={{ $json.pagination.next }}",
"operation": "notEqual",
"value2": "={{ null }}"
}
]
}
},
"id": "056c1de9-134b-4904-bb2f-1db736a074e8",
"name": "IF2",
"type": "n8n-nodes-base.if",
"typeVersion": 1,
"position": [
900,
520
]
},
{
"parameters": {
"operation": "aggregateItems",
"aggregate": "aggregateAllItemData"
},
"id": "cfb0bd6a-a44e-4401-b5c9-ac3c8d4fdc0a",
"name": "Item Lists",
"type": "n8n-nodes-base.itemLists",
"typeVersion": 1,
"position": [
1920,
500
]
},
{
"parameters": {
"jsCode": "// Loop over input items and add a new field\n// called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.pagination.next = null;\n}\n\nreturn $input.all();"
},
"id": "3f010c51-7276-4af9-bb14-acba7bdf1503",
"name": "Code1",
"type": "n8n-nodes-base.code",
"typeVersion": 1,
"position": [
-180,
440
]
},
{
"parameters": {
"table": {
"__rl": true,
"value": "CRM.RespondIO",
"mode": "name"
},
"columns": "id,firstName,lastName,phone,email,language,profilePic,locale,countryCode,status,created_at",
"options": {
"ignore": true
}
},
"id": "6859eab1-624b-4cd6-85da-7f7ba80f7f8a",
"name": "MySQL",
"type": "n8n-nodes-base.mySql",
"typeVersion": 1,
"position": [
820,
140
],
"credentials": {
"mySql": {
"id": "5",
"name": "CRM"
}
}
},
{
"parameters": {
"operation": "executeQuery",
"query": "Select * from RespondIO"
},
"id": "f6459205-8fb9-4e9d-afc0-d5cccfc0754b",
"name": "MySQL1",
"type": "n8n-nodes-base.mySql",
"typeVersion": 1,
"position": [
1440,
420
],
"credentials": {
"mySql": {
"id": "5",
"name": "CRM"
}
}
},
{
"parameters": {
"batchSize": 1,
"options": {}
},
"id": "fc01b9e6-ee27-48e4-b64a-060eef257b21",
"name": "Split In Batches",
"type": "n8n-nodes-base.splitInBatches",
"typeVersion": 1,
"position": [
600,
140
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "// Add a new field called 'myNewField' to the\n// JSON of the item\nvar mysql = require('mysql');\n\nvar con = mysql.createConnection({\n host: \"localhost\",\n user: \"yourusername\",\n password: \"yourpassword\"\n});\n\ncon.connect(function(err) {\n if (err) throw err;\n console.log(\"Connected!\");\n var names = \"id, firstName, lastName, phone, email, language, profilePic, locale, countryCode, status, custom_fields, tags, assignee, created_at\";\n var values = \"'\"+item[0].json.id+\"','\"+item[0].json.firstName+\"','\"+item[0].json.lastName+\"','\"+item[0].json.phone+\"','\"+item[0].json.email+\"','\"+item[0].json.language+\"','\"+item[0].json.profilePic+\"','\"+item[0].json.locale+\"','\"+item[0].json.countryCode+\"','\"+item[0].json.status+\"','\"+item[0].json.custom_fields+\"','\"+item[0].json.tags+\"','\"+item[0].json.assignee+\"','\"+item[0].json.created_at+\"'\";\n var sql = \"Insert into `CRM`.`RespondIO` (\"+names+\") VALUES (\"+values+\")\";\n con.query(sql, function (err, result) {\n if (err) throw err;\n console.log(\"1 record inserted\");\n });\n});\n\n\nreturn $input.item;"
},
"id": "e22b59a5-62ad-4041-bba6-85cf3b012ff8",
"name": "Code2",
"type": "n8n-nodes-base.code",
"typeVersion": 1,
"position": [
820,
0
]
},
{
"parameters": {
"mode": "runOnceForEachItem",
"jsCode": "return {json:{pagination: {next:item.json.pagination.next}}}"
},
"id": "9a808968-2b64-465f-a9de-9b2b926626a0",
"name": "Code3",
"type": "n8n-nodes-base.code",
"typeVersion": 1,
"position": [
1600,
820
]
}
],
"connections": {
"Item Lists1": {
"main": [
[
{
"node": "Split In Batches",
"type": "main",
"index": 0
}
]
]
},
"HTTP Request": {
"main": [
[
{
"node": "HTTP Request1",
"type": "main",
"index": 0
},
{
"node": "Code1",
"type": "main",
"index": 0
}
]
]
},
"HTTP Request1": {
"main": [
[
{
"node": "NoOp",
"type": "main",
"index": 0
}
]
]
},
"IF": {
"main": [
[
{
"node": "MySQL1",
"type": "main",
"index": 0
}
],
[
{
"node": "IF1",
"type": "main",
"index": 0
}
]
]
},
"Code": {
"main": [
[
{
"node": "Item Lists",
"type": "main",
"index": 0
}
]
]
},
"Execute Workflow Trigger": {
"main": [
[
{
"node": "HTTP Request",
"type": "main",
"index": 0
}
]
]
},
"NoOp": {
"main": [
[
{
"node": "IF2",
"type": "main",
"index": 0
},
{
"node": "Item Lists1",
"type": "main",
"index": 0
}
]
]
},
"IF1": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
],
[
{
"node": "Code3",
"type": "main",
"index": 0
}
]
]
},
"IF2": {
"main": [
[
{
"node": "IF",
"type": "main",
"index": 0
}
]
]
},
"Code1": {
"main": [
[
{
"node": "NoOp",
"type": "main",
"index": 0
}
]
]
},
"MySQL1": {
"main": [
[
{
"node": "Code",
"type": "main",
"index": 0
}
]
]
},
"Split In Batches": {
"main": [
[
{
"node": "MySQL",
"type": "main",
"index": 0
}
]
]
},
"Code3": {
"main": [
[
{
"node": "HTTP Request1",
"type": "main",
"index": 0
}
]
]
}
}
}
Information on your n8n setup
- **n8n version:**0.217.1
- **Database you’re using (default: SQLite): MariaDB
- **Running n8n with the execution process [own(default), main]:own
- **Running n8n via [Docker, npm, n8n.cloud, desktop app]:Docker
