Read a ND-JSON stream from the browser or from nodejs

browser-ndjson-stream-reader.js

/* FOR THE BROWSER
Utility function to read a ND-JSON HTTP stream.
`processLine` is a function taking a JSON object. It will be called with each element of the stream.
`response` is the result of a `fetch` request.
See usage example in the next file.
*/

const readStream = processLine => response => {
  const stream = response.body.getReader();
  const matcher = /\r?\n/;
  const decoder = new TextDecoder();
  let buf = '';

  const loop = () =>
    stream.read().then(({ done, value }) => {
      if (done) {
        if (buf.length > 0) processLine(JSON.parse(buf));
      } else {
        const chunk = decoder.decode(value, {
          stream: true
        });
        buf += chunk;

        const parts = buf.split(matcher);
        buf = parts.pop();
        for (const i of parts.filter(p => p)) processLine(JSON.parse(i));
        return loop();
      }
    });

  return loop();
}

nodejs-ndjson-stream-reader.js

/* FOR NODEJS
Utility function to read a ND-JSON HTTP stream.
`processLine` is a function taking a JSON object. It will be called with each element of the stream.
`response` is the result of a `fetch` request.
See usage example in the next file.
*/
const readStream = processLine => response => {
  const matcher = /\r?\n/;
  const decoder = new TextDecoder();
  let buf = '';
  return new Promise((resolve, fail) => {
    response.body.on('data', v => {
      const chunk = decoder.decode(v, { stream: true });
      buf += chunk;

      const parts = buf.split(matcher);
      buf = parts.pop();
      for (const i of parts.filter(p => p)) processLine(JSON.parse(i));
    });
    response.body.on('end', () => {
      if (buf.length > 0) processLine(JSON.parse(buf));
      resolve();
    });
    response.body.on('error', fail);
  });
};

usage.js

/* From browser or nodejs alike */

const stream = fetch('https://lichess.org/api/tv/feed');
// or any other ND-JSON endpoint such as:
// const stream = fetch('https://lichess.org/api/games/user/neio',{headers:{Accept:'application/x-ndjson'}});

const onMessage = obj => console.log(obj);
const onComplete = () => console.log('The stream has completed');

stream
  .then(readStream(onMessage))
  .then(onComplete);

@rodolphonetto I'm trying to do the same, did you find any way to do this with react native ?

@konpa Sorry but no... I started again using Ionic and there I had success

@rodolphonetto Ok, then I'll keep looking for a solution :) Thanks for your answer!

Just as information, in case it could help someone with the same problem, I managed to make it work with react native (only tested on iOS for now) using acostalima/react-native-polyfill-globals and the web-streams-polyfill, inexorabletash/text-encoding and react-native-community/fetch dependencies.

Hi @konpa Is it possible to make a little exemple of how did you done? It can be helpfull for begginers

Thanks!

Sure.

Install dependancies

npm install react-native-polyfill-globals text-encoding react-native-fetch-api web-streams-polyfill

MyComponentScreen.native.tsx (polyfill are not needed/working for web)

import { polyfill as polyfillEncoding } from 'react-native-polyfill-globals/src/encoding';
import { polyfill as polyfillReadableStream } from 'react-native-polyfill-globals/src/readable-stream';
import { polyfill as polyfillFetch } from 'react-native-polyfill-globals/src/fetch';

polyfillEncoding();
polyfillReadableStream();
polyfillFetch();

[...]

const readStream = (processLine) => (response) => {
  // same as browser-ndjson-stream-reader.js above
  [...]
}

const stream = fetch(`https://lichess.org/api/board/game/stream/${gameId}`, {
  reactNative: { textStreaming: true },
  method: 'GET',
  headers: {
    Authorization: `Bearer ${accessToken}`,
    Accept: 'application/x-ndjson',
  },
});

const onMessage = (obj) => console.log(obj);
const onComplete = () => console.log('The stream has completed');

stream
  .then(readStream(onMessage))
  .then(onComplete);