three.js + headless.gl rendering with texturing

README.md

Getting the code

git clone https://gist.github.com/08be90a2f21205062ccc.git

Executing the code

$ npm install # maybe npm start will take care of it but just in case
$ npm start && open out.png

> [email protected] start /Users/bsergean/src/offscreen_sample
> coffee offscreen_sample.coffee

THREE.WebGLRenderer 71
THREE.WebGLRenderer: TypeError: Object #<Object> has no method 'addEventListener'
THREE.WebGLRenderer: OES_texture_float extension not supported.
THREE.WebGLRenderer: OES_texture_float_linear extension not supported.
THREE.WebGLRenderer: OES_texture_half_float extension not supported.
THREE.WebGLRenderer: OES_texture_half_float_linear extension not supported.
THREE.WebGLRenderer: OES_standard_derivatives extension not supported.
THREE.WebGLRenderer: OES_element_index_uint extension not supported.
THREE.WebGLRenderer: EXT_texture_filter_anisotropic extension not supported.
Image written: out.png

Those warnings are harmless for our test case, but might be problematic for some folks. Support for extension is planned and coming -> stackgl/headless-gl#5

If you are on Linux you will need Xvfb. One way to do it:

$ xvfb-run -s "-ac -screen 0 1280x1024x24” node_modules/.bin/coffee offscreen_sample.coffee

More infos here -> https://github.com/stackgl/headless-gl#how-can-headless-gl-be-used-on-a-headless-linux-machine

Inspecting the output

Tada ! You just created an image thanks to OpenGL and many awesome libraries. How cool is that. Now open the output image. On a Mac you can just do that:

open out.png

Here's how it should look like: ->

Bummer, the sample is in coffee-script

npm run compile

This will compile the .coffee file to javascript and print it in your terminal. It's almost the same as the .coffee.

offscreen_sample.coffee

# The required node modules
THREE = require('three')
PNG   = require('pngjs').PNG
gl    = require("gl")()
fs    = require('fs')

render = (dataTexture) ->
    # Parameters (the missing one is the camera position, see below)
    width  = 600
    height = 400
    path   = 'out.png'
    png    = new PNG({ width: width, height: height })

    # THREE.js business starts here
    scene = new THREE.Scene()

    # camera attributes
    VIEW_ANGLE = 45
    ASPECT = width / height
    NEAR = 0.1
    FAR  = 100

    # set up camera
    camera = new THREE.PerspectiveCamera(VIEW_ANGLE, ASPECT, NEAR, FAR)

    scene.add(camera)
    camera.position.set(0, 2, 2)
    camera.lookAt(scene.position)

    # mock object, not used in our test case, might be problematic for some workflow
    canvas = new Object()

    # The width / height we set here doesn't matter
    renderer = new THREE.WebGLRenderer({
        antialias: true,
        width: 0,
        height: 0,
        canvas: canvas, # This parameter is usually not specified
        context: gl     # Use the headless-gl context for drawing offscreen
    })

    # add some geometry
    geometry = new THREE.BoxGeometry( 1, 1, 1 )

    # add a material; it has to be a ShaderMaterial with custom shaders for now 
    # this is a work in progress, some related link / issues / discussions
    #
    # https://github.com/stackgl/headless-gl/issues/26
    # https://github.com/mrdoob/three.js/pull/7136
    # https://github.com/mrdoob/three.js/issues/7085
    material = new THREE.ShaderMaterial()

    material.vertexShader = '''
    varying vec2 vUv;
    
    void main() {
        vUv = uv;
        gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
    }
    '''
    material.fragmentShader = '''
    uniform sampler2D dataTexture;
    varying vec2 vUv;

    void main() {
        gl_FragColor = texture2D(dataTexture, vUv);
    }
    '''
    material.uniforms =
        dataTexture: { type: "t", value: dataTexture }

    # Create the mesh and add it to the scene
    cube     = new THREE.Mesh(geometry, material)
    scene.add(cube)

    # Let's create a render target object where we'll be rendering
    rtTexture = new THREE.WebGLRenderTarget(
        width, height, {
            minFilter: THREE.LinearFilter,
            magFilter: THREE.NearestFilter,
            format: THREE.RGBAFormat
    })

    # render
    renderer.render(scene, camera, rtTexture, true)

    # read render texture into buffer
    gl = renderer.getContext()

    # create a pixel buffer of the correct size
    pixels = new Uint8Array(4 * width * height)

    # read back in the pixel buffer
    gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels)

    # lines are vertically flipped in the FBO / need to unflip them
    for j in [0...height]
        for i in [0...width]
            k = j * width + i
            r = pixels[4*k]
            g = pixels[4*k + 1]
            b = pixels[4*k + 2]
            a = pixels[4*k + 3]

            m = (height - j + 1) * width + i
            png.data[4*m]     = r
            png.data[4*m + 1] = g
            png.data[4*m + 2] = b
            png.data[4*m + 3] = a

    # Now write the png to disk
    stream = fs.createWriteStream(path)
    png.pack().pipe stream

    stream.on 'close', () ->
        # We're done !!
        console.log("Image written: #{ path }")

png = new PNG
stream = fs.createReadStream 'turtle.png'
stream.pipe png

png.on 'parsed', () =>
    console.log 'parsed !', width, height

    width  = png.width
    height = png.height
    data   = png.data

    console.log typeof(pixels)
    dataTexture = new THREE.DataTexture(data, width, height,
                                        THREE.RGBAFormat )
    dataTexture.needsUpdate = true

    render(dataTexture)

package.json

{
  "name": "offscreen-sample",
  "version": "1.0.0",

  "scripts": {
    "start": "coffee offscreen_sample.coffee",
    "compile": "coffee -c -b offscreen_sample.coffee && cat offscreen_sample.js"
  },

  "dependencies": {
    "three": "latest",
    "pngjs": "latest", 
    "gl": "latest"
  },

  "devDependencies": {
    "coffee-script": "latest"
  }
}

turtle.png

Hi.

When I try to use the example code I get this error:
THREE.WebGLProgram: shader error: 1282 gl.VALIDATE_STATUS false gl.getPRogramInfoLog not linked
THREE.WebGLProgram: gl.getProgramInfoLog()not linked

It seems that the following condition in three.js (line 24556) is always false:

if ( _gl.getProgramParameter( program, _gl.LINK_STATUS ) === false ) {

		THREE.error( 'THREE.WebGLProgram: shader error: ' + _gl.getError(), 'gl.VALIDATE_STATUS', _gl.getProgramParameter( program, _gl.VALIDATE_STATUS ), 'gl.getPRogramInfoLog', programLogInfo );

	}

Here the javascript code:

const width   = 64
const height  = 64
var gl = require('gl')(width, height, { preserveDrawingBuffer: true })
var THREE = require('three')

var scene = new THREE.Scene()

// camera attributes
const VIEW_ANGLE = 45
const ASPECT = width / height
const NEAR = 0.1
const FAR  = 100

// set up camera
var camera = new THREE.PerspectiveCamera(VIEW_ANGLE, ASPECT, NEAR, FAR)

scene.add(camera)
camera.position.set(0, 2, 2)
camera.lookAt(scene.position)

var canvas = new Object()
//canvas.addEventListener = function(){}

var renderer = new THREE.WebGLRenderer({
	antialias: true,
	width: 0,
	height: 0,
	canvas: canvas, 
	context: gl     
})

// add geoometry
var geometry = new THREE.BoxGeometry( 1, 1, 1 )


var vertexShader = `
    void main() {
        gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 );
    }
`;

var fragmentShader = `
    void main() {
        gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
    }
`;

var  material = new THREE.ShaderMaterial(
	{
		vertexShader: vertexShader,
		fragmentShader: fragmentShader
	}
)

var cube     = new THREE.Mesh(geometry, material)
scene.add(cube)
 
var  rtTexture = new THREE.WebGLRenderTarget(
	width, height, {
		minFilter: THREE.LinearFilter,
		magFilter: THREE.NearestFilter,
		format: THREE.RGBAFormat
})

console.log("1");
renderer.render(scene, camera, rtTexture, true)

//renderer.render(scene, camera);
console.log("2");

// read render texture into buffer
var gl = renderer.getContext()
console.log("3");
// create a pixel buffer of the correct size
var pixels = new Uint8Array(4 * width * height)

// read back in the pixel buffer
gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels)
console.log("pixels", pixels);

btw: the lines which is failing is
renderer.render(scene, camera, rtTexture, true)

Sorry I haven't used this in years (probably 5) so unless you pinned the version of Three.js it might very likely have broken.

I used version 73 back then I believe.

Actually it was THREE.WebGLRenderer 71

None of these solutions worked for me, and I spent around 2-3 days on it. I have also been trying on a AWS container image, which offered me more flexibility, and yet no success. However, I was able to make it work using Puppeteer!

My advice for everyone is:

• Use an AWS Lambda container. Something like public.ecr.aws/lambda/nodejs:12 will work well, but is just matter of preference
• Use Puppeteer to open a local, but extremely simple, HTML that loads your bundled (I used Webpack) local Javascript file that contains the ThreeJS library

If you have questions, let me know.

I did that thing in 2015, I was using centos7 back then.

I think with docker it should work but I have not put any effort into this for a long time.

Maybe with Docker and an official build of CentOS 7 + all of the NPM packages with their respective versions from the time you wrote this should work, but I didn't have more time to fiddle with it, so I accepted working with Puppeteer. At the end, their headless Chrome API is very light and fast, so it didn't seem like a bad route either.

Btw, the reason I said "official CentOS 7 build" is because AWS Lambda is derived from CentOS 7, however it differs in some parts and may lack certain libraries, so that could be the reason why in didn't work for me. To be more specific, it was because my headless-gl context was null, as pointed by this issue.