Wolfenstein .Pages files (idtech5 virtualtextures)

[m0xf]

Images are compressed with HDphoto, not Jpeg XR. This specs are similar, but has some difference. And yes, duffuse map using YCoCg color space.

[volfin]

Images are compressed with HDphoto, not Jpeg XR. This specs are similar, but has some difference. And yes, duffuse map using YCoCg color space.

Ok, I'll have a look then. I've saved files as both JXR (Jpeg XR) and WDP (Windows HD Photo) using a popular imaging app, and the formats seemed identical. But i'll take your word for it they are not the same. At least from what I observed with this template file I created for JXR.

Code: Select all

//--------------------------------------
//--- 010 Editor v5.0 Binary Template
//
// File:
// Author: Volfin
// Revision:
// Purpose: Decode Jpeg XR headers
//--------------------------------------

local ulong n;
local int i;
local ulong ifd_offset;
local ubyte x;
local ushort Num_Components;
local ushort NumLPQPs;
local ushort NumHPQPs;
local ubyte NumBands;
local ulong valueNumIndexTableEntries;
local uint NumMBInTile[100];

local ulong NumMBInCurrentTile;
local ulong NumBandsOfPrimary;
local ubyte IsCurrPlaneAlphaFlag=FALSE;
local uint64 Subsequent_Bytes;
local ulong valueAdditionalBytes;
local ulong iBytes;
local ubyte lastflag;
local ushort Num_Vert_Tiles_Minus1=0;
local ushort Num_Horiz_Tiles_Minus1=0; 
local uint64 IndexOffsetTile[16];

void do_coded_tiles()
{
   local uint64 current_pointer;
   current_pointer=FTell();

   Printf("staring do_coded_tiles() FilePos:%ld\n",FTell());
    // 8.7.1
    if (IMG_HDR.Freq_Mode_Codestream_Flag == 0)
    {
        for(n=0;n<(Num_Horiz_Tiles_Minus1+1)*(Num_Vert_Tiles_Minus1+1);n++)
        {
            NumMBInCurrentTile=NumMBInTile[n];
            FSeek(IndexOffsetTile[n]+current_pointer);
            TILE_SPATIAL();
        }
    }
    else
    {
        for(n=0;n<(Num_Horiz_Tiles_Minus1+1)*(Num_Vert_Tiles_Minus1+1);n++)
        {
            NumMBInCurrentTile=NumMBInTile[n];
            FSeek(IndexOffsetTile[n*NumBandsOfPrimary]+current_pointer);
            TILE_DC();
        }        
    }

    if (NumBandsOfPrimary > 1)
    {
        for(n=0;n<(Num_Horiz_Tiles_Minus1+1)*(Num_Vert_Tiles_Minus1+1);n++)
        {
            NumMBInCurrentTile=NumMBInTile[n];
            FSeek(IndexOffsetTile[n*NumBandsOfPrimary+1]+current_pointer);
            TILE_LOWPASS();
        }            
    }

    if (NumBandsOfPrimary > 2)
    {
        for(n=0;n<(Num_Horiz_Tiles_Minus1+1)*(Num_Vert_Tiles_Minus1+1);n++)
        {
            NumMBInCurrentTile=NumMBInTile[n];
            FSeek(IndexOffsetTile[n*NumBandsOfPrimary+2]+current_pointer);
            TILE_HIGHPASS();
        }            
    }

    if (NumBandsOfPrimary > 3)
    {
        for(n=0;n<(Num_Horiz_Tiles_Minus1+1)*(Num_Vert_Tiles_Minus1+1);n++)
        {
            NumMBInCurrentTile=NumMBInTile[n];
            FSeek(IndexOffsetTile[n*NumBandsOfPrimary+3]+current_pointer);
            TILE_FLEXBITS();
        }            
    }
}

void TILE_SPATIAL()
{
    Printf("Tile_Spatial\n");
    ulong Tile_Startcode:24;
    uchar Arbitrary_Byte;    
}
///////////////////////////////////////////////////////
void TILE_DC()
{
    Printf("Tile_DC\n");
    ulong Tile_Startcode:24;
    uchar Arbitrary_Byte;
    IsCurrPlaneAlphaFlag=FALSE;
    TILE_HEADER_DC();
    if (IMG_HDR.Alpha_Image_Plane_Flag)
    {
        IsCurrPlaneAlphaFlag = TRUE;
        TILE_HEADER_DC( ); //8.7.4
    }
    for (n = 0; n < NumMBInCurrentTile; n++)
    {
        IsCurrPlaneAlphaFlag = FALSE;
        MB_DC( ); //8.7.11
        if (IMG_HDR.Alpha_Image_Plane_Flag)
        {
            IsCurrPlaneAlphaFlag = TRUE;
            MB_DC( ); //8.7.11
        }
    }

    //while (!IS_BYTE_ALIGNED( ))
        //BYTE_ALIGNMENT_BIT
}

void TILE_HEADER_DC()
{
    if (IMG_PL_HDR.DC_Image_Plane_Uniform_Flag == FALSE)
    {
        DC_QP( ); //8.4.22
    }    
}
//////////////////////////////////////////////////////////
void TILE_LOWPASS()
{
    Printf("Tile_Lowpass\n");
    ulong Tile_Startcode:24;
    uchar Arbitrary_Byte;

    IsCurrPlaneAlphaFlag=FALSE;
    if (IMG_PL_HDR.Bands_Present!= 3) /*  DCONLY - BANDS_PRESENT of primary image plane */
        TILE_HEADER_LOWPASS( ); // 8.7.6

    if (IMG_HDR.Alpha_Image_Plane_Flag)
    {
        IsCurrPlaneAlphaFlag = TRUE;
        if (IMG_PL_HDR.Bands_Present != 3) /* DCONLY - BANDS_PRESENT of alpha image plane */
            TILE_HEADER_LOWPASS( ); //8.7.6
    }

    for (n = 0; n < NumMBInCurrentTile; n++)
    {
        IsCurrPlaneAlphaFlag = FALSE;
        if (IMG_PL_HDR.Bands_Present != 3) // DCONLY 
        { /* BANDS_PRESENT of primary image plane */
            if (NumLPQPs > 1 && USE_DC_QP_FLAG == FALSE)
            LP_QP_INDEX[n] = DECODE_QP_INDEX(NumLPQPs); /* primary image plane */
            MB_LP( ); //8.7.16.1
        }
        if (IMG_HDR.Alpha_Image_Plane_Flag)
        {
            IsCurrPlaneAlphaFlag = TRUE;
            if (IMG_PL_HDR.Bands_Present != 3) // DCONLY
            { /* BANDS_PRESENT of alpha image plane */
                if (NumLPQPs > 1 && USE_DC_QP_FLAG == FALSE)
                    LP_QP_INDEX[n] = DECODE_QP_INDEX(NumLPQPs);  /* alpha image plane */

                MB_LP( );
            }
        }
    }

    //while (!IS_BYTE_ALIGNED( ))
      //  BYTE_ALIGNMENT_BIT
}

void TILE_HEADER_LOWPASS()
{
    if (IMG_PL_HDR.DC_Image_Plane_Uniform_Flag == FALSE)
    {
        ubyte USE_DC_QP_FLAG:1;
        if (USE_DC_QP_FLAG)
        {
            NumLPQPs = 1;
        }
        else
        {
            ubyte NUM_LP_QPS_MINUS1:4;
            NumLPQPs = NUM_LP_QPS_MINUS1+1;
            LP_QP();
        }
    }    
}
/////////////////////////////////////////////////////////
void TILE_HIGHPASS()
{
    Printf("Tile_HighPass\n");
    ulong Tile_Startcode:24;
    uchar Arbitrary_Byte;
}
/////////////////////////////////////////////////////////
void TILE_FLEXBITS()
{
    Printf("Tile_FLEXBITS\n");
    ulong Tile_Startcode:24;
    uchar Arbitrary_Byte;
}
/////////////////////////////////////////////////////////
void DC_QP()
{
    if (Num_Components != 1)
    {
        ubyte Component_Mode:2;
        if (Component_Mode == 0) //UNIFORM
        {
            ubyte DC_QUANT:8;
        }
        else if (Component_Mode == 1) //SEPARATE
        {
            ubyte DC_QUANT_LUMA:8;
            ubyte DC_QUANT_CHROMA:8;
        }
        else if (Component_Mode == 2) //INDEPENDENT
        {
            for (i=0;i<Num_Components;i++)
            {
                ubyte DC_QUANT_CH:8;
            }         
        }    
    }    
}

void LP_QP()
{
    for (i = 0; i < NumLPQPs; i++)
    {
        if (Num_Components != 1)
        {
            ubyte COMPONENT_MODE:2;

            if (COMPONENT_MODE == 0) // UNIFORM
            {
                ubyte LP_QUANT:8;
            }
            else if (COMPONENT_MODE == 1)//SEPARATE
            {
                ubyte LP_QUANT_LUMA:8;
                ubyte LP_QUANT_CHROMA:8;
            }
            else if (COMPONENT_MODE == 2)//INDEPENDENT
            {
                for (i = 0; i < Num_Components; i++)
                {
                    ubyte LP_QUANT_CH:8;
                }
            }
        }
    }
}

void HP_QP()
{
    for (i = 0; i < NumHPQPs; i++)
    {
        if (Num_Components != 1)
        {
            ubyte COMPONENT_MODE:2;

            if (COMPONENT_MODE == 0) // UNIFORM
            {
                ubyte HP_QUANT:8;
            }
            else if (COMPONENT_MODE == 1)//SEPARATE
            {
                ubyte HP_QUANT_LUMA:8;
                ubyte HP_QUANT_CHROMA:8;
            }
            else if (COMPONENT_MODE == 2)//INDEPENDENT
            {
                for (i = 0; i < Num_Components; i++)
                {
                    ubyte HP_QUANT_CH:8;
                }
            }
        }
    }
}

void do_IMG_PL_HDR()
{
  // 8.4.1
    struct IMG_PLANE_HDR
    {
        ubyte Internal_CLR_Format:3;
        ubyte Scaled_flag:1;
        ubyte Bands_Present:4;
        
        if (Bands_Present == 0)
            NumBands=4;
        else if (Bands_Present == 1)
            NumBands=3;
        else if (Bands_Present == 2)
            NumBands=2;
        else if (Bands_Present == 3)
            NumBands=1;
        if (IsCurrPlaneAlphaFlag==0)
        {
            NumBandsOfPrimary = NumBands;
        }
    
        if (Internal_CLR_Format == 1 || Internal_CLR_Format == 2 || Internal_CLR_Format == 3)
        {
            Num_Components=3;
            // YUV420 or YUV422 or YUV444
            if (Internal_CLR_Format == 1 || Internal_CLR_Format == 2)
            {
                // YUV420/YUV422
                ubyte Reserved_E:1;
                ubyte Chroma_Centering_X:3;                
            }
            else
            {
                ubyte Reserved_F:4;
            }


            if (Internal_CLR_Format == 1)
            {
                // YUV420
                ubyte Reserved_G:1;
                ubyte Chroma_Centering_Y:3;
            }
            else
            {
                ubyte Reserved_H:4;
            }
        }
        else if (Internal_CLR_Format == 6)
        {
            // NComponent 
            ubyte Num_Components_Minus1:4;
            if (Num_Components_Minus1 == 0xF)
            {
                ushort Num_Components_Extended_Minus16:12;
                Num_Components=Num_Components_Extended_Minus16+16;
            }
            else
            {
                ubyte Reserved_H2:4;
                Num_Components=Num_Components_Minus1+1;
            }
            
        }
        else if (Internal_CLR_Format == 4) //YUVK
            Num_Components=4;
        else if (Internal_CLR_Format == 0) // YONLY
            Num_Components=1;
      
        // Bitdepth from IMG_HDR.Output_Bitdepth
        if (IMG_HDR.Output_Bitdepth == 2 || IMG_HDR.Output_Bitdepth == 3 || IMG_HDR.Output_Bitdepth == 6)
        {
            ubyte Shift_Bits;
        }
        if (IMG_HDR.Output_Bitdepth == 7)
        {
            ubyte LEN_Mantissa;
            byte EXP_Bias;
        }
    
        ubyte DC_Image_Plane_Uniform_Flag:1;
    
        if (DC_Image_Plane_Uniform_Flag)
        {
            DC_QP();
        }

        if (Bands_Present != 3) // DCONLY
        {
            ubyte Reserved_I:1;
            ubyte LP_Image_Plane_Uniform_Flag:1;
            if (LP_Image_Plane_Uniform_Flag)
            {
                NumLPQPs=1;
                LP_QP();
            }
            if (Bands_Present != 2) // NOHIGHPASS
            {
                ubyte Reserved_J:1;
                ubyte HP_Image_Plane_Uniform_Flag:1;
                if (HP_Image_Plane_Uniform_Flag)
                {
                    NumHPQPs=1;
                    HP_QP();
                }        
            }        
        }
        
//        ubyte IsByteAligned;
//        ubyte dummy;  
    } IMG_PL_HDR;
}

uint64 VLW_ESC()
{
    struct VLW_Bytes
    {
        ubyte FIRST_BYTE:8;
        if (FIRST_BYTE <0xFB)
        {
            ubyte SECOND_BYTE:8;
            return(FIRST_BYTE*256+SECOND_BYTE);
        }
        else if (FIRST_BYTE==0xFB)
        {
            ulong FOUR_BYTES:32;
            return FOUR_BYTES;
        }
        else if (FIRST_BYTE==0xFC)
        {
            uint64 EIGHT_BYTES:64;
            return EIGHT_BYTES;
        }
        else return 0;
    } VLW_BYTES;
}

ulong Profile_Level_Info()
{
    local ulong mmBytes=0;
    local ulong iLast;
    lastflag=0;
    for (iLast=0;iLast==0;iLast=lastflag)
    {
        struct Profile 
        {
            ubyte PROFILE_IDC;
            ubyte LEVEL_IDC;
            ushort Reserved_L:15;
            ubyte LAST_FLAG:1;
            lastflag=LAST_FLAG;
        } PROFILE;
        mmBytes=mmBytes+4;
    }
    return mmBytes;
}

//////////////////////////////////////
LittleEndian();
BitfieldLeftToRight();
BitfieldDisablePadding();


struct Header { 
    char fixed_header_II[2]; 
    ubyte fixed_header_0XBC;
    ubyte version_id;
    long first_IFD_Offset;    
  } HEADER;

FSeek(HEADER.first_IFD_Offset);

struct Image_File_Directory {
    short NUM_ENTRIES;
    for (i=0;i<NUM_ENTRIES;i++)
    {
        struct IFD_ENTRY {
            short FIELD_TAG;
            short ELEMENT_TYPE;
            long NUM_ELEMENTS;
            long VALUES_OR_OFFSET;
        } IFDE;
    }
    long NEXT_IFD;
}IFD;

BigEndian();

struct Image_Header {
    char GDI_SIG[8];
    ubyte Reserved_B:4;
    ubyte Hard_tiling_flag:1;
    ubyte Reserved_C:3;
    ubyte Tiling_Flag:1;
    ubyte Freq_Mode_Codestream_Flag:1;
    ubyte Spatial_Xfrm_Subordinate:3;
    ubyte Index_Table_Flag:1;
    ubyte Overlap_Mode:2;
    ubyte Short_header_flag:1;
    ubyte Long_Word_Flag:1;
    ubyte Windowing_Flag:1;
    ubyte Trim_Flexbits_Flag:1;
    ubyte Reserved_D:1;
    ubyte Red_Blue_not_Swapped_Flag:1;
    ubyte Premultiplied_Alpha_Flag:1;
    ubyte Alpha_Image_Plane_Flag:1;
    ubyte Output_CLR_FMT:4;
    ubyte Output_Bitdepth:4;

    if (Short_header_flag)
    {
        ushort Width_Minus1;
        ushort Height_Minus1;
    }
    else
    {
        ulong Width_Minus1;
        ulong Height_Minus1;
    }

    if (Tiling_Flag)
    {
        // numbers are in 12 bits for some reason
        ushort Num_Vert_Tiles_Minus1t:12;
        ushort Num_Horiz_Tiles_Minus1t:12;
        Num_Vert_Tiles_Minus1=Num_Vert_Tiles_Minus1t;
        Num_Horiz_Tiles_Minus1=Num_Horiz_Tiles_Minus1t;

        for (i=0;i<Num_Vert_Tiles_Minus1;i++)
        {
            if (Short_header_flag)
            {
                ubyte Tile_Width_In_MB;
            }
            else
            {
                ushort Tile_Width_In_MB;
            }
        }

        for (i=0;i<Num_Horiz_Tiles_Minus1;i++)
        {
            if (Short_header_flag)
            {
                ubyte Tile_Height_In_MB;
            }
            else
            {
                ushort Tile_Height_In_MB;
            }
        }

        
    }


    if (Windowing_Flag)
    {
        ubyte TOP_MARGIN:6;
        ubyte LEFT_MARGIN:6;
        ubyte BOTTOM_MARGIN:6;
        ubyte RIGHT_MARGIN:6;
    }

} IMG_HDR;

do_IMG_PL_HDR();

if (IMG_HDR.Alpha_Image_Plane_Flag)
{
    IsCurrPlaneAlphaFlag=TRUE;
    do_IMG_PL_HDR();
}

if (IMG_HDR.Index_Table_Flag)
{
    struct INDEX_TABLE_TILES
    {
        if (IMG_HDR.Freq_Mode_Codestream_Flag == 0)
        {
            valueNumIndexTableEntries=1;
        }
        else
        {
            valueNumIndexTableEntries=(1*NumBandsOfPrimary);
        }
        ushort Index_Table_Startcode;
        
        Printf("valueNumIndexTableEntries:%ld\n",valueNumIndexTableEntries);

        
        for (n = 0; n < valueNumIndexTableEntries; n++)
        {   
            IndexOffsetTile[n]=VLW_ESC();         
            Printf("Tile offset %d: %ud\n",n,IndexOffsetTile[n]);
 
        }
        
    }IDX_T_TILES;
}

Subsequent_Bytes=VLW_ESC();
Printf("Subsequent Bytes:%d\n",Subsequent_Bytes);
if (Subsequent_Bytes > 0)
{
    //  etc.. See 8.2.1
    iBytes=Profile_Level_Info();
    Printf("iBytes:%d\n",iBytes);
    valueAdditionalBytes=Subsequent_Bytes - iBytes;
    Printf("valueAdditionalBytes:%d\n",valueAdditionalBytes);
    for(iBytes=0;iBytes < valueAdditionalBytes; iBytes++)
        ubyte RESERVED_A_BYTE;
}

// 8.7.1
do_coded_tiles();

It breaks the header info down as far as the CODED_BLOCKS, but it's still not 100 complete/tested.

[volfin]

I finally bought the Rage Toolkit to see if it would offer any insight. And I did find it useful to create a test dataset. I made 4 textures, which are 512x512, with a simple descriptive text on each layer, saying what it is. (Diffuse/Specular/Normal/Bump).

I then created a simple map with only this texture used, and built the map so it's own virtual texture file would be created. During the build process, an intermediate file is created, with the *.cpuvmtr extention. And this intermediate file seems to be the raw texture before compression. By examining it one can see what is stored in the *.pages file. I found they actually store 11 layers, in this order:

Diffuse Red
Diffuse Green
Diffuse Blue
Specular Red
Specular Green
Specular Blue
Normal Red
Normal Green
Normal Blue
Power Map Red
Blank Layer (white)

I'm including this intermediate file, along with my source images, and the compressed megatexture output, as a test dataset. Having a known imput will make working backward easier. As the texture contained within is my own orignal work, I hereby give full permission to copy and distribute.

http://home.comcast.net/~volfin/Known%20Image.rar

You can open the cpuvmtr as a raw file, 512x512 dimensions, 11 channels interleaved, with a 212 byte header.

[Wobble]

[out]

[volfin]

I finally bought the Rage Toolkit to see if it would offer any insight.

What else can you do with the Rage Took Kit?

I read it has a MD6 Model Editor. Can it be used to reverse the md6model format?
I still don't know how bone-vertex assignments work.

No, it only allows editing maps, creating walls and ground and doors, that type of thing. And you can place 'entities' and 'objects' which are the predefined models included in Rage. But it doesn't really have a way to introduce new models. In fact the only way i could get it to use my custom texture was to replace the pre-existing TGAs for an existing texture with my own. There's no way to simply tell it to use custom textures either. Very, very limited.

EDIT: now that I look harder, it does have a model viewer, that does read in the md6model. There's no import and there's no type of export except for something called "Transformed LWO" which doesn't actually seem to do anything (no files produced). I don't see it being much help for understanding the format.

[luxox18]

I extracted some virtual textures from the game with AMD CODEXL but the software not support resolutions over 8192.

this is a normal texture from Infantry46 character (4096x4096)

http://www.mediafire.com/view/oa2a7r3pp ... level0.png

and this is {Diffuse?, maybe damaged} texture from the same character (4096x4096)

http://www.mediafire.com/view/0t343kw2w ... level0.png

apply zoom in both images or download the file for see the detail.


AMD codexl is a profiling tool for Directx, opencl and opengl , maybe can help to know how the game transcode the textures with the CPU-GPU analysis and other useful options

[volfin]

I extracted some virtual textures from the game with AMD CODEXL but the software not support resolutions over 8192.

this is a normal texture from Infantry46 character (4096x4096)

http://www.mediafire.com/view/oa2a7r3pp ... level0.png

and this is {Diffuse?, maybe damaged} texture from the same character (4096x4096)

http://www.mediafire.com/view/0t343kw2w ... level0.png

apply zoom in both images or download the file for see the detail.

AMD codexl is a profiling tool for Directx, opencl and opengl , maybe can help to know how the game transcode the textures with the CPU-GPU analysis and other useful options

Very Interesting. This isn't exactly how I imagined it, but it does make sense I guess. To the engine it's not important that the tiles be in order, only that they be present in memory. Likely pulled straight from the Quadtree in the order stored.

This confirms the page block is 128x128 pixels as inferred from the presentations, that's something at least.

You can actually see how the texture Mip-Maps on the lower tiles, one 128x128 block contains a reduced-size version of several blocks for far-away items. Very cool.

[m0xf]

I found way to extract textures. To do this, I load the game exe in memory and call the decoding function. Tiles overlapped 4px. Effective tile size is 120x120.

Textures really large, therefore I split it to 15630x15630 images.
To use this tool you need get "Reloaded" exe v1.0.0.1 (md5: e0617f6cbbe81cfdcfbedb7f6f01a557).

[volfin]

Impressive. I'll have to give this a go tomorrow.

I got the EXE with the matching MD5, and tried it. But for me the program fails with a BEX64 error. tragic.

[m0xf]

Yes, I see some error on clean system. Try to disable ASLR (disable_aslr.reg and reboot). You can enable it again after using vtex.

[volfin]

Yes, I see some error on clean system. Try to disable ASLR (disable_aslr.reg and reboot). You can enable it again after using vtex.

Yes! That got it. Working great. I tip my hat to you sir.

[luxox18]

sorry for the noob question but what do you mean with " I load the game exe in memory and call the decoding function" ?
can you explain how to make this?

EDIT......

Solved! thanks volfin

[sarzamin]

sorry for the noob question but what do you mean with " I load the game exe in memory and call the decoding function" ?
can you explain how to make this?

EDIT......

Solved! thanks volfin

Hi, Could you tell me How can I load the game exe in memory and call the decoding function? "

[luxox18]

put the vtex.exe tool in the main directory of the game, run the tool, load the pages file and add the output directory

example: vtex.exe C:\Blah-Blah...\virtualpages\chunk2_vmtr.pages C:\base

then use irfanview for edit and view the DDS file

[sarzamin]

put the vtex.exe tool in the main directory of the game, run the tool, load the pages file and add the output directory

example: vtex.exe C:\Blah-Blah...\virtualpages\chunk2_vmtr.pages C:\base

then use irfanview for edit and view the DDS file

Thanks . but it' didn't worked for me .

this is my directory of the game :
D:\Wolfenstein - The New Order

i put vtex.exe in the main directory (beside WolfNewOrder_x64.exe )
and then run this Command :

"vtex.exe D:\Wolfenstein - The New Order\virtualtextures\chunk1_vmtr.pages D:\Wolfenstein - The New Order\EXTRACTED "

also try this :

"vtex.exe virtualtextures\chunk1_vmtr.pages EXTRACTED "

But it's not worked

I used to this kind of command line tools before...but i don't know why it's not worked. did i something wrong?