[FIXED] LibGDX / OpenGL Stencil Buffer masking not working

Issue

I'm using LibGDX to render 2d fog-of-war type functionality. This involves drawing a dark rectangle over the entire map with transparent holes in it where you can see the map below. I'm attempting to use OpenGl stencil buffer to create the circular masks, but I can't seem to get the logic correct.

The code below correctly draws the dark rectangle (the fog) but the circular masks are not being stenciled. i.e. the entire map is dark.

 batch.end();

    Gdx.gl.glClear(GL_STENCIL_BUFFER_BIT);
    Gdx.gl.glEnable(GL20.GL_STENCIL_TEST);
    Gdx.gl.glColorMask(false, false, false, false);
    //always write the clipped holes to the stencil
    Gdx.gl.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
    Gdx.gl.glStencilMask(0xFF);
    Gdx.gl.glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);

    shapeRenderer.begin(ShapeRenderer.ShapeType.Filled);
    shapeRenderer.setProjectionMatrix(gameUi.camera.combined);
   
     //test circle
    shapeRenderer.circle(0, 0, 1000, 100);

    shapeRenderer.end();


    Gdx.gl.glEnable(GL20.GL_BLEND);
    Gdx.gl.glBlendFunc(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA);
    Gdx.gl.glColorMask(true, true, true, true);
    Gdx.gl.glEnable(GL20.GL_STENCIL_TEST);
    //only draw the fog of war where stencil is 0
    Gdx.gl.glStencilFunc(GL_EQUAL, 0x0, 0xffffffff);
    Gdx.gl.glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);


    //draw fog of war
    shapeRenderer.begin(ShapeRenderer.ShapeType.Filled);
    shapeRenderer.setProjectionMatrix(gameUi.camera.combined);
    shapeRenderer.setColor(radarFog);
    int dimension = gameUi.mapConfig.getDimension();
    shapeRenderer.rect(-dimension,-dimension,dimension*3,dimension*3);
    shapeRenderer.end();

    Gdx.gl.glDisable(GL20.GL_STENCIL_TEST);

    batch.begin();

Solution

I was finally able to get this to work with stencils. There were some issues with the stencil ops I was using but the code below is working for me.

        //enable stencil and depth testing
        gl.glEnable(GL_STENCIL_TEST);
        gl.glEnable(GL_DEPTH_TEST);
        //enable blending because our fog has alpha
        Gdx.gl.glEnable(GL20.GL_BLEND);
        Gdx.gl.glBlendFunc(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA);
        //stencil op which writes to stencil when test passes
        gl.glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
        //clear stencil buffer
        gl.glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
        //always pass the stencil test and mask to accept all values
        gl.glStencilFunc(GL_ALWAYS, 1, 0xFF);
        gl.glStencilMask(0xFF);

        //draw our cut out circles
        drawCircles();

        //change stencil function to only match where stencil was written
        gl.glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
        //change mask so nothing is written to stencil
        gl.glStencilMask(0x00);


        drawFog();

        //disable features
        gl.glDisable(GL_STENCIL_TEST);
        gl.glDisable(GL_DEPTH_TEST);
        gl.glDisable(GL_BLEND);

Answered By - The Shoe Shiner