Forget it, I was a bit sleepy yesterday, should have posted the code anyway.
In Core/VideoCommon/VertexLoader.cpp, replace method UpdateBoundingBox() with this one:
void LOADERDECL UpdateBoundingBox()
{
if (!PixelEngine::bbox_active)
return;
// reset videodata pointer
VertexManager::s_pCurBufferPointer = s_bbox_pCurBufferPointer_orig;
// copy vertex pointers
memcpy(VertexManager::s_pCurBufferPointer, s_bbox_vertex_buffer, 12);
VertexManager::s_pCurBufferPointer += 12;
// We must transform the just loaded point by the current world and projection matrix - in software.
// Then convert to screen space and update the bounding box.
float p[3] = {s_bbox_vertex_buffer[0], s_bbox_vertex_buffer[1], s_bbox_vertex_buffer[2]};
const float *world_matrix = (float*)xfmem + MatrixIndexA.PosNormalMtxIdx * 4;
// We need to get the raw projection values for the bounding box calculation
// to work properly. That means, no projection hacks!
float *rawProjection = xfregs.projection.rawProjection;
float proj_matrix[16];
switch (xfregs.projection.type)
{
case GX_PERSPECTIVE:
proj_matrix[0] = rawProjection[0]; proj_matrix[1] = 0.0f; proj_matrix[2] = rawProjection[1]; proj_matrix[3] = 0.0f;
proj_matrix[4] = 0.0f; proj_matrix[5] = rawProjection[2]; proj_matrix[6] = rawProjection[3]; proj_matrix[7] = 0.0f;
proj_matrix[8] = 0.0f; proj_matrix[9] = 0.0f; proj_matrix[10] = rawProjection[4]; proj_matrix[11] = rawProjection[5];
proj_matrix[12] = 0.0f; proj_matrix[13] = 0.0f; proj_matrix[14] = -1.0f; proj_matrix[15] = 0.0f;
break;
case GX_ORTHOGRAPHIC:
proj_matrix[0] = rawProjection[0]; proj_matrix[1] = 0.0f; proj_matrix[2] = 0.0f; proj_matrix[3] = rawProjection[1];
proj_matrix[4] = 0.0f; proj_matrix[5] = rawProjection[2]; proj_matrix[6] = 0.0f; proj_matrix[7] = rawProjection[3];
proj_matrix[8] = 0.0f; proj_matrix[9] = 0.0f; proj_matrix[10] = rawProjection[4]; proj_matrix[11] = rawProjection[5];
proj_matrix[12] = 0.0f; proj_matrix[13] = 0.0f; proj_matrix[14] = 0.0f; proj_matrix[15] = 1.0f;
break;
default:
ERROR_LOG(VIDEO, "Unknown projection type: %d", xfregs.projection.type);
}
float t[3];
t[0] = p[0] * world_matrix[0] + p[1] * world_matrix[1] + p[2] * world_matrix[2] + world_matrix[3];
t[1] = p[0] * world_matrix[4] + p[1] * world_matrix[5] + p[2] * world_matrix[6] + world_matrix[7];
t[2] = p[0] * world_matrix[8] + p[1] * world_matrix[9] + p[2] * world_matrix[10] + world_matrix[11];
float o[3];
o[0] = t[0] * proj_matrix[0] + t[1] * proj_matrix[1] + t[2] * proj_matrix[2] + proj_matrix[3];
o[1] = t[0] * proj_matrix[4] + t[1] * proj_matrix[5] + t[2] * proj_matrix[6] + proj_matrix[7];
o[2] = t[0] * proj_matrix[12] + t[1] * proj_matrix[13] + t[2] * proj_matrix[14] + proj_matrix[15];
o[0] /= o[2];
o[1] /= o[2];
// Max width is 608, while max height is 480
// Could this be register dependant? Seems to work this way, though
o[0] = (o[0] + 1.0f) * 304.0f;
o[1] = (1.0f - o[1]) * 240.0f;
if (o[0] < PixelEngine::bbox[0]) PixelEngine::bbox[0] = (u16) std::max(0.0f, o[0]);
if (o[0] > PixelEngine::bbox[1]) PixelEngine::bbox[1] = (u16) o[0];
if (o[1] < PixelEngine::bbox[2]) PixelEngine::bbox[2] = (u16) std::max(0.0f, o[1]);
if (o[1] > PixelEngine::bbox[3]) PixelEngine::bbox[3] = (u16) o[1];
}
This might also fix many minor issues with Super Paper Mario, and hopefully also the hang on world 6-1. Needs testing...
Updated by 
Updated by 
Updated by 
Updated by 
Updated by 
Updated by 