Paginating API Results

When using the API, Dexi limits responses to 200 items per page. To display more than 200 items, you can use pagination to load additional results. Refer to the code examples below to see how this is done using Java and Javascript. If you need additional assistance configuring pagination, please contact our support team.

GET /runs
Java

import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;


public class RunsClientTestScript {

    private static final String ACCESS_KEY = "<access_key>";
    private static final String ACCOUNT = "<account_id>";
    private static final String ACCEPT = "application/json";

    public static void main(String[] args) throws IOException, InterruptedException {
        getAllRecords();
    }

    private static List<Object> getAllRecords() throws IOException, InterruptedException {
        List<Object> allRecords = new ArrayList();
        int offset = 0;
        int totalRows = -1;
        do {
            String resultStr = getRuns(offset);
            Map<String, Object> resultMap = convertJSONStrToMap(resultStr);
            List<Object> rows = (List<Object>) resultMap.get("rows");
            for (Object object : rows) {
                System.out.println(object);
                allRecords.add(object);
            }
            //update variable only on first iteration
            if (totalRows == -1)
                totalRows = (int) resultMap.get("totalRows");
            //update offset so that you get new page on next iteration
            offset = allRecords.size();
            Thread.sleep(300);
        } while (offset < totalRows);

        return allRecords;
    }

    private static String getRuns(int offset) throws IOException {
        String urlStr = "https://api.dexi.io/runs?limit=200&offset=" + offset;
        URL url = new URL(urlStr);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        conn.setRequestProperty("X-DexiIO-Access", ACCESS_KEY);
        conn.setRequestProperty("X-DexiIO-Account", ACCOUNT);
        conn.setRequestProperty("Accept", ACCEPT);

        if (conn.getResponseCode() != 200) {
            throw new RuntimeException("Failed : HTTP Error code : " + conn.getResponseCode() + " - " + conn.getResponseMessage());
        }
        InputStreamReader inputStream = new InputStreamReader(conn.getInputStream());
        String strCurrentLine;
        StringBuilder resultStr = new StringBuilder();
        try (BufferedReader in = new BufferedReader(inputStream)) {
            while ((strCurrentLine = in.readLine()) != null) {
                resultStr.append(strCurrentLine);
            }
        }
        conn.disconnect();
        return resultStr.toString();
    }

    private static Map<String, Object> convertJSONStrToMap(String result) throws IOException {
        ObjectMapper mapper = new ObjectMapper();
        return mapper.readValue(result, Map.class);
    }
}

JavaScript

const https = require('https');
const queryString = require('querystring')

let totalRows = -1;
let collectedItems = [];


// API request to get runs
const getRunsRows = (options) => {
  return new Promise((resolve, reject) => {
    response = https.request(options, async (res) => {
      let response_body = '';
      res.on('data', (d) => {
        response_body += d; // collect the data
      });
      res.on('end', () => {
        resolve(response_body); //resolve the promise on request end
      });
      res.on('error', (err) => {  
        reject(err);
      });
    });
    response.end()
  });
};

// get all runs
const getAllRows = async () => {
  let totalRows = -1; //
  let rowsRetrieved = []; //collected data
  let cursor = 0; // current possition
  do {
      const requestOptions = {
      //constuct the headers .
      method: 'GET',
      hostname: 'api.dexi.io',
      path: '/runs?limit=200&offset=' + cursor ,
      headers: {
        'X-DexiIO-Access': 'ACCESS_KEY',
        'X-DexiIO-Account': 'ACCOUNT',
        'Accept': 'ACCEPT'
      },
    };

    let response = await getRunsRows(requestOptions);

    let res = JSON.parse(response);

    if (totalRows < 0) {
      //set the max rows to know when to jump out of the loop.
      totalRows = res.totalRows;
    }
    rowsRetrieved = [...rowsRetrieved, ...res.rows];
    cursor = rowsRetrieved.length;
  } while (cursor < totalRows); //repeat while there are more rows to retrieve.
  return rowsRetrieved;
};

async function main() {
 let returnData =  await getAllRows();
 console.log(returnData);
}

main();

POST /datasets/{dataSetId}/rows
Java

import com.fasterxml.jackson.databind.ObjectMapper;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
import java.util.Map;

public class MainApp {

    private static final String DATASET_ID = "PLACEHOLDER";
    private static final String PAYLOAD = "{\n" +
            "\t\"offset\": 0,\n" +
            "\t\"limit\": 2\n" +
            "}";

    private static final String ACCESS_KEY = "PLACEHOLDER";
    private static final String ACCOUNT = "PLACEHOLDER";
    private static final String ACCEPT = "application/json";
    private static final String CONTENT_TYPE = "application/json";

    public static void main(String[] args) throws IOException {
        String resultStr = getDatasetRows(DATASET_ID);
        Map<String, Object> resultMap = convertJSONStrToMap(resultStr);
        List rows = (List) resultMap.get("rows");
        for (Object object : rows) {
            System.out.println(object);
        }

    }

    private static String getDatasetRows(String datasetId) throws IOException {
        String urlStr = "https://api.dexi.io/datasets/${datasetId}/rows".replace("${datasetId}", datasetId);
        URL url = new URL(urlStr);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("X-DexiIO-Access", ACCESS_KEY);
        conn.setRequestProperty("X-DexiIO-Account", ACCOUNT);
        conn.setRequestProperty("Accept", ACCEPT);
        conn.setRequestProperty("Content-Type", CONTENT_TYPE);
        //conn.setRequestProperty("Content-Length", String.valueOf(PAYLOAD.length()));
        conn.setDoOutput(true);
        try (OutputStream os = conn.getOutputStream()) {
            byte[] input = PAYLOAD.getBytes("utf-8");
            os.write(input, 0, input.length);
        }
        if (conn.getResponseCode() != 200) {
            throw new RuntimeException("Failed : HTTP Error code : " + conn.getResponseCode());
        }
        InputStreamReader inputStream = new InputStreamReader(conn.getInputStream());
        BufferedReader result = new BufferedReader(inputStream);
        String strCurrentLine;
        String resultStr = "";
        while ((strCurrentLine = result.readLine()) != null) {
            resultStr += strCurrentLine;
        }
        conn.disconnect();
        return resultStr;
    }

    private static Map<String, Object> convertJSONStrToMap(String result) throws IOException {
        ObjectMapper mapper = new ObjectMapper();
        Map<String, Object> map = mapper.readValue(result, Map.class);
        return map;
    }
}

JavaScript

const https = require('https');

let totalRows = -1;
let collectedItems = [];

const getPayload = (offset) => {
    return {
        offset,
        limit: 200,
        query:{}
    }
}


// API request to get dataset
const getDatasetRows = (options, payload) => {
    return new Promise((resolve, reject) => {
        response = https.request(options, async (res) => {
            let response_body = '';
            res.on('data', (d) => {
                response_body += d; // collect the data 
            });
            res.on("end", () => {
                resolve(response_body);//resolve the promise on request end
            })
            res.on('error', (err) => {
                reject(err);
            })

        })
        response.write(JSON.stringify(payload))
    })
}

// get all dataset rows
const getAllRows = async (datasetId) => {
    let totalRows = -1; // 
    let rowsRetrieved = [];//collected data
    let cursor = 0;// current possition 
    do{
        let payload = getPayload(cursor);//get new payload with new offset.

        const requestOptions = {//constuct the headers .
            method: "POST",
            hostname: "api.dexi.io",
            path: `/datasets/${datasetId}/rows`,
            headers: {
                'X-DexiIO-Access': '<Access-key>',
                'X-DexiIO-Account': '{Account-id}',
                'Accept': 'application/json',
                'Content-Type': 'application/json',
                'Content-Length': JSON.stringify(payload).length
            }
        }
    
        let response = await getDatasetRows(requestOptions, payload);
        let res = JSON.parse(response);
        if (totalRows < 0) { //set the max rows to know when to jump out of the loop.
            totalRows = res.totalRows 
        }
        rowsRetrieved = [...rowsRetrieved, ...res.rows];
        cursor = rowsRetrieved.length;
    }while(cursor<totalRows);//repeat while there are more rows to retrieve.
    return rowsRetrieved
}

async function main() {
    let dataset = await getAllRows("<datasetId>");
    // the variable dataset contains all the rows in the dataset
}

main();