Reading variables from file does not last

Hi,
I have set up a workflow “template” that is reading values from a json-file with the “ReadBinaryFile” node (/home/node/.n8n/{{$workflow.name}}.json)+ Move Binary Data → json. As a third node i read values from a postgres-database using values pulled from the json-file. This part works.

Next i loop the postgres-result in 10 items batches with a 1 sec sleep-node against a api that is limited to 10 requests per second. I this loop i use again values read by the Move Binary Data → json, but the vaules are empty. For some reason the values get cleared during execution, and the workflow fails…

I wonder if i’m doing something wrong, and loose the variables because of this?

Sorry for this extremely clumsy question… n8n is quite new to me, so expressing myself in a decent way is still “work in progress” :slight_smile:

Can you please share the workflow so that I can have a better look? Just select everything copy and then paste it here.

Here is my workflow:

{
  "nodes": [
    {
      "parameters": {},
      "name": "Start",
      "type": "n8n-nodes-base.start",
      "typeVersion": 1,
      "position": [
        500,
        410
      ]
    },
    {
      "parameters": {
        "authentication": "headerAuth",
        "requestMethod": "POST",
        "url": "=https://{{$node[\"Settings\"].json[\"api\"][\"base\"]}}/{{$node[\"Settings\"].json[\"api\"][\"workspaceid\"]}}/{{$node[\"Settings\"].json[\"api\"][\"endpoint\"]}}",
        "jsonParameters": true,
        "options": {
          "fullResponse": true
        },
        "bodyParametersJson": "={{$json[\"body\"]}}"
      },
      "name": "GetClockifyID",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 1,
      "position": [
        1710,
        410
      ],
      "credentials": {
        "httpHeaderAuth": "ClockifyHeaderAuth"
      },
      "continueOnFail": true
    },
    {
      "parameters": {
        "functionCode": "return new Promise((resolve) => {\n  setTimeout(() => {\n    resolve(items);\n  }, $node[\"Settings\"].json[\"batchsettings\"][\"waittime\"] );\n});\n\n"
      },
      "name": "Wait1",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1410,
        410
      ]
    },
    {
      "parameters": {
        "functionCode": "// Parse ClockifyGetIDResponse\nlet arr = [];\nfunction reverseString(str) {\n    let newString = \"\";\n    for (let i = str.length - 1; i >= 0; i--) {\n        newString += str[i];\n    }\n    return newString;\n}\nitems.forEach(function (elem1,index) {\n  // if(elem1.json.error) {\n  //   arr.push({});\n  //   arr[arr.length-1].json={};\n  //   arr[arr.length-1].json.statusCode=elem1.json.error.statusCode\n  // }\n\n  if(elem1.json.body) {\n    // productID in beginning of name\n    let hansaid = elem1.json.body.name.split(\" \",1)[0]\n    // productID in end of name\n    // let hansaid = reverseString(reverseString(elem1.json.body.name).split(\" \",1)[0]);\n \n    let clockifyid = elem1.json.body.id\n    arr.push({});\n    arr[arr.length-1].json={};\n    arr[arr.length-1].json.clockifyid=clockifyid\n    arr[arr.length-1].json.hansaid=hansaid\n    arr[arr.length-1].json.name=elem1.json.body.name\n    arr[arr.length-1].json.jsonfield=JSON.stringify(elem1.json.body)\n    arr[arr.length-1].json.statusCode=elem1.json.body.statusCode\n    arr[arr.length-1].json.body=\"{\\\"name\\\":\\\"\" + hansaid + \"\\\"}\"\n    console.log(clockifyid + \" \" + hansaid)\n    arr[arr.length-1].json.statuscode=elem1.json.statusCode\n\n  }\n});\nreturn arr;\n"
      },
      "name": "ParseGetClockifyID",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1860,
        410
      ],
      "continueOnFail": true
    },
    {
      "parameters": {
        "functionCode": "//Finalize alldata\nlet arr = [];\nitems.forEach(function (elem1,index) {\n  elem1.json.allData.forEach(function (elem2,index) {\n    arr.push({});\n    arr[arr.length-1].json={};\n    arr[arr.length-1].json.hansaid=elem2.hansaid\n    arr[arr.length-1].json.name=JSON.stringify(elem2.name)\n    arr[arr.length-1].json.clockifyid=elem2.clockifyid\n    arr[arr.length-1].json.jsonfield=JSON.stringify(elem2)\n  });\n});\nreturn arr;\n\n"
      },
      "name": "FinalizeAllData",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1690,
        590
      ]
    },
    {
      "parameters": {
        "operation": "executeQuery",
        "query": "=UPDATE public.{{$node[\"Settings\"].json[\"clockify2hansa\"][\"hansa\"]}}\nSET clockifyid = '{{$json[\"clockifyid\"]}}',\njsonfield = cast('{{$json[\"jsonfield\"]}}' as json)\nWHERE hansaid = '{{$json[\"hansaid\"]}}';",
        "additionalFields": {}
      },
      "name": "UpdateClockifyID2Postgres",
      "type": "n8n-nodes-base.postgres",
      "typeVersion": 1,
      "position": [
        1890,
        590
      ],
      "alwaysOutputData": true,
      "notesInFlow": true,
      "credentials": {
        "postgres": "Postgresql"
      }
    },
    {
      "parameters": {
        "functionCode": "// Parse postgres results\nlet arr = [];\nitems.forEach(function(elem1,index) {\n  let hansaid = elem1.json.hansaid\n  let name = elem1.json.name\n  arr.push({});\n  arr[arr.length-1].json={}\n  arr[arr.length-1].json.hansaid=hansaid\n  arr[arr.length-1].json.name=JSON.stringify(elem1.json.name)\n  arr[arr.length-1].json.body=(\"{\\\"name\\\":\" + JSON.stringify(name) + \"}\")\n});\nreturn arr;\n"
      },
      "name": "ParsePostgres",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1570,
        410
      ]
    },
    {
      "parameters": {
        "conditions": {
          "boolean": [
            {
              "value1": true,
              "value2": "={{$node[\"SplitInBatches4\"].context[\"noItemsLeft\"]}}"
            }
          ]
        }
      },
      "name": "IfBatchLeft",
      "type": "n8n-nodes-base.if",
      "position": [
        1310,
        610
      ],
      "typeVersion": 1
    },
    {
      "parameters": {
        "batchSize": "={{$node[\"Settings\"].json[\"batchsettings\"][\"batchamount\"]}}",
        "options": {}
      },
      "name": "SplitInBatches4",
      "type": "n8n-nodes-base.splitInBatches",
      "position": [
        1260,
        410
      ],
      "typeVersion": 1
    },
    {
      "parameters": {
        "operation": "executeQuery",
        "query": "=SELECT hansaid,name\nFROM public.{{$node[\"Settings\"].json[\"clockify2hansa\"][\"hansa\"]}}\nwhere  clockifyid is null\norder by hansaid",
        "additionalFields": {}
      },
      "name": "GetPostgresProducts",
      "type": "n8n-nodes-base.postgres",
      "typeVersion": 1,
      "position": [
        1050,
        410
      ],
      "credentials": {
        "postgres": "Postgresql"
      }
    },
    {
      "parameters": {
        "functionCode": "// ParseAllData\nconst allData = []\nlet counter = 0;\ndo {\n  try {\n    const items = $items(\"ParseGetClockifyID\", 0, counter).map(item => item.json);\n    allData.push.apply(allData, items);\n  } catch (error) {\n    return [{json: {allData}}];  \n  }\n\n  counter++;\n} while(true);\n"
      },
      "name": "CombineBatchLoop",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1510,
        590
      ]
    },
    {
      "parameters": {
        "options": {}
      },
      "name": "Settings",
      "type": "n8n-nodes-base.moveBinaryData",
      "typeVersion": 1,
      "position": [
        860,
        410
      ]
    },
    {
      "parameters": {
        "filePath": "=/home/node/.n8n/{{$workflow.name}}.json"
      },
      "name": "ReadConfigFile",
      "type": "n8n-nodes-base.readBinaryFile",
      "typeVersion": 1,
      "position": [
        680,
        410
      ]
    }
  ],
  "connections": {
    "Start": {
      "main": [
        [
          {
            "node": "ReadConfigFile",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "GetClockifyID": {
      "main": [
        [
          {
            "node": "ParseGetClockifyID",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Wait1": {
      "main": [
        [
          {
            "node": "ParsePostgres",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "ParseGetClockifyID": {
      "main": [
        [
          {
            "node": "IfBatchLeft",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "FinalizeAllData": {
      "main": [
        [
          {
            "node": "UpdateClockifyID2Postgres",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "ParsePostgres": {
      "main": [
        [
          {
            "node": "GetClockifyID",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "IfBatchLeft": {
      "main": [
        [
          {
            "node": "CombineBatchLoop",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "SplitInBatches4",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "SplitInBatches4": {
      "main": [
        [
          {
            "node": "Wait1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "GetPostgresProducts": {
      "main": [
        [
          {
            "node": "SplitInBatches4",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "CombineBatchLoop": {
      "main": [
        [
          {
            "node": "FinalizeAllData",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Settings": {
      "main": [
        [
          {
            "node": "GetPostgresProducts",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "ReadConfigFile": {
      "main": [
        [
          {
            "node": "Settings",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}

Do you have an example of how the setting node returns the data? Does it return more than one item?

Sure, this is what it returns (I cleaned some sensitive data). The postgresql-part was ment to be used for postgres credentials, but that was also empty when executing the workflow…

[
    {
        "api": {
            "base": "api.clockify.me/api/v1/workspaces",
            "endpoint": "tags",
            "workspaceid": "xxxxxxxxxxxxxxxxxxxxxxxx"
        },
        "batchsettings": {
            "batchamount": "10",
            "waittime": "1200"
        },
        "clockify2hansa": {
            "clockify": "products",
            "hansa": "products"
        },
        "postgresql": {
            "psqldb": "db",
            "psqlhost": "host",
            "psqlpass": "pass",
            "psqluser": "user"
        }
    }
]

Ok, in the last node. Can you change {{$node["Settings"].json["clockify2hansa"]["hansa"]}} for {{$item(0).$node["Settings"].json["clockify2hansa"]["hansa"]}} and try again?

That did not work. It worked though inside the loop, where the http request node had trouble getting the values too… The last node just get’s an [not found]

Are you getting a not found in the Postgre node?

Yes, the very last node.:frowning:

I finally got this solved:
$items(“Settings”)[0].json.clockify2hansa.hansa
The issue is though still there. A value that is uses via the expression builder where i refer to a specific node, get’s cleared up after it has been used once in eg a loop. The second round has the values cleared…