Loop through pagination in HTTP node

Fragen
,
1

Describe the issue/error/question

Hello, Thanks in advance for your help. It seems everytime I try something in n8n, I get a new set of questions to solve. :frowning: I hope to turn things around soon.

I need to query a custom API , fetch some data such as products’ sku, quantity in stock, and other related data. After I’ve queried the API, I will merge fields and discard other, to put it into a spreadsheet.

However, I cannot get to loop through the API pagination. How can I do that?

What is the error message (if any)?

No erros, just empty result.

Please share the workflow

This is the output returned by the API from a GET to the inventory endpoint. The important part that I need to modify to loop is in the value “next” , to page=3 , page=4 and so on. The last page has the value of “null”. That is why in the IF node I am setting this as the condition to continue looping.

{
    "count": 24979,
    "next": "http://mycustomapi/stock/?page=2",
    "previous": null,
    "results": [
        {
            "cve_art": "09157948",
            "descr": "No. De Parte Remplazado por: 9157948",
            "lin_prod": "9INTE",
            "con_serie": "N",
            "uni_med": "pz",
            "uni_emp": 1.0,
            "ctrl_alm": null,
            "tiem_surt": 0,
            "stock_min": 0.0,
            "stock_max": 0.0,
            "tip_costeo": "P",
            "num_mon": 1,
            "fch_ultcom": null,
            "comp_x_rec": 0.0,
            "fch_ultvta": null,
            "pend_surt": 0.0,
            "exist": 0.0,
            "costo_prom": 0.0,
            "ult_costo": 0.0,
            "cve_obs": 0,
            "tipo_ele": "P",
            "uni_alt": "pz",
            "fac_conv": 1.0,
            "apart": 0.0,
            "con_lote": "N",
            "con_pedimento": "N",
            "peso": 0.0,
            "volumen": 0.0,
            "cve_esqimpu": 1,
            "cve_bita": null,
            "vtas_anl_c": 0.0,
            "vtas_anl_m": 0.0,
            "comp_anl_c": 0.0,
            "comp_anl_m": 0.0,
            "prefijo": null,
            "talla": null,
            "color": null,
            "cuent_cont": null,
            "cve_imagen": null,
            "blk_cst_ext": "N",
            "status": "A",
            "man_ieps": "S",
            "apl_man_imp": 1,
            "cuota_ieps": 0.0,
            "apl_man_ieps": "C",
            "uuid": "5A14CDFA-5E57-4B32-AC0F-6AE4A9BB520A",
            "version_sinc": "2022-05-26T15:39:12.019000Z",
            "version_sinc_fecha_img": null,
            "cve_prodserv": "25172000",
            "cve_unidad": "H87",
            "edo_publ_ml": null,
            "cve_publ_ml": null,
            "condicion_ml": null,
            "tipo_publ_ml": null,
            "modo_envio_ml": null,
            "largo_ml": null,
            "alto_ml": null,
            "ancho_ml": null,
            "envio_ml": null,
            "categ_ml": null,
            "campos_categ_ml": null,
            "disponible_publ": null,
            "cve_cate_ml": null,
            "last_update_ml": null,
            "f_crea_ml": null,
            "imagen_ml": null,
            "en_catalogo": null,
            "id_catalogo": null,
            "titulo_ml": null,
            "mat_peli": null
        },
       {some 20 more products}
    ]
}

Information on your n8n setup

  • n8n version: n8n@0.178.1
  • Database you’re using (default: SQLite): yes
  • Running n8n with the execution process [own(default), main]: no idea
  • Running n8n via [Docker, npm, n8n.cloud, desktop app]: docker
2

The example below should do it.

3

Worked like a charm! Thank you.
I liked how you set up the Function 1 to define the endpoint, not only this scenario works, it is also easy to understand.

4

Great that it worked. Have fun.

5

Hello, sorry I jump in this conversation, but I dont see the solution “example below” , is it missing ? thanks !!

6

Hi @Vincent_Wasteels - there’s a temporary issue with previewing workflows on the forums, but you can find the JSON code here:

{
  "nodes": [
    {
      "parameters": {},
      "name": "Start",
      "type": "n8n-nodes-base.start",
      "typeVersion": 1,
      "position": [
        180,
        300
      ]
    },
    {
      "parameters": {
        "conditions": {
          "string": [
            {
              "value1": "={{$node[\"Fetch API\"].json[\"next\"]}}",
              "value2": "={{null}}"
            }
          ]
        }
      },
      "name": "IF",
      "type": "n8n-nodes-base.if",
      "typeVersion": 1,
      "position": [
        1280,
        300
      ]
    },
    {
      "parameters": {
        "url": "={{ $node[\"Function1\"].json[\"next\"] }}",
        "options": {}
      },
      "name": "Fetch API",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 1,
      "position": [
        700,
        300
      ],
      "credentials": {
        "httpHeaderAuth": {
          "id": "4",
          "name": "SOL LOGIN"
        }
      }
    },
    {
      "parameters": {
        "fieldToSplitOut": "results",
        "options": {}
      },
      "name": "Item Lists",
      "type": "n8n-nodes-base.itemLists",
      "typeVersion": 1,
      "position": [
        960,
        300
      ]
    },
    {
      "parameters": {
        "keepOnlySet": true,
        "values": {
          "string": [
            {
              "name": "next",
              "value": "={{$node[\"Fetch API\"].json[\"next\"]}}"
            }
          ]
        },
        "options": {}
      },
      "name": "Set",
      "type": "n8n-nodes-base.set",
      "typeVersion": 1,
      "position": [
        980,
        600
      ]
    },
    {
      "parameters": {
        "functionCode": "\nconsole.log(items[0].json.next)\n\n\nreturn {\n  next: items[0].json.next || \"https://fake-api-pagination-m3j5j7bsccr1.runkit.sh/api\",\n}"
      },
      "name": "Function1",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        440,
        300
      ]
    },
    {
      "parameters": {
        "functionCode": "const allData = []\n\nlet counter = 0;\ndo {\n  try {\n    const items = $items(\"Item Lists\", 0, counter).map(item => item.json);\n    allData.push.apply(allData, items);\n  } catch (error) {\n    return allData.map(data => ({ json: data}));  \n  }\n\n  counter++;\n} while(true);\n"
      },
      "name": "Merge executions",
      "type": "n8n-nodes-base.function",
      "typeVersion": 1,
      "position": [
        1540,
        280
      ]
    }
  ],
  "connections": {
    "Start": {
      "main": [
        [
          {
            "node": "Function1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "IF": {
      "main": [
        [
          {
            "node": "Merge executions",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "Set",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Fetch API": {
      "main": [
        [
          {
            "node": "Item Lists",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Item Lists": {
      "main": [
        [
          {
            "node": "IF",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Set": {
      "main": [
        [
          {
            "node": "Function1",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Function1": {
      "main": [
        [
          {
            "node": "Fetch API",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  }
}
`