Finally worked out a method that supports full rotations on a quad. I guess you could adapt it to any object, but for now it just works for a quad.
I rayhit each triangle and worked out the barycentric coordinates to calculate the UV's which I converted into texture coordinates. This became difficult when rotating since vertex positions don't update with rotation. So I used the object direction vector to move the points then do the calculations. I think the code still needs a bit of work, especially when it comes to working out the texture coordinates between faces, but least it's working.
// thanks TheComet and Neuro Fuzzy for the help
type point2
x as integer
y as integer
endtype
type vector3
x as float
y as float
z as float
endtype
global _point2 as point2
global _vector3 as vector3
global _oldTick as dword
global _newTick as dword : _newTick = timer() //hitimer()
global _delta as float : global _elasped as float
sync on : sync rate 0
autocam off : hide mouse
// create bitmap for are object
global _texCoordinates as point2
global _texSize as integer : _texSize = 64
create bitmap 1,_texSize,_texSize
get image 1,0,0,_texSize,_texSize
set current bitmap 0
// create plain for are interactive surface
global _objectPos as vector3
global _objectAng as vector3
make object plain 1, 6.4, 6.4, 1
texture object 1, 1
set object filter 1, 0
set object light 1, 0
SC_setupComplexObject 1 , 0 , 2
// create a memblock of the object
// this is used to access the positions of each vertex later on
make mesh from object 1, 1
make memblock from mesh 1, 1
delete mesh 1
// get some header data so we know where to find the position data in the memblock
global _memVertSize as dword
_memVertSize = memblock dword( 1, 4 )
// setup camera
global _cameraPos as vector3
global _cameraAng as vector3
position camera 0, 0, -10
point camera 0, 0, 0
// store centre of screen
global _screenWidthHalf as integer
global _screenHeightHalf as integer
_screenWidthHalf = screen width() / 2
_screenHeightHalf = screen height() / 2
// use limb offset of dummy mesh to work out direction vector
global _dummyMesh as integer
_dummyMesh = find free object()
make object triangle _dummyMesh, 0,0,0 ,0,0,0 ,0,0,0
exclude object on _dummyMesh
// misc
global _rayHitPosition as vector3
global _rayCollision as byte
global _faceHit as integer
a as vector3 : b as vector3 : c as vector3
do
// update frame tick
_oldTick = _newTick
_newTick = timer() //hitimer()
_delta = (_newTick-_oldTick)*0.001
// update mouse look
xrotate camera 0, wrapvalue(camera angle x(0) + mousemovey() / 5.0)
yrotate camera 0, wrapvalue(camera angle y(0) + mousemovex() / 5.0)
// update camera
getCameraPosition() : _cameraPos = _vector3
getCameraAngle() : _cameraAng = _vector3
// update object
rotate object 1,wrapvalue(object angle x(1)+(10*_delta)),wrapvalue(object angle y(1)+(10*_delta)),wrapvalue(object angle z(1)+(20*_delta))
SC_UpdateObject 1
getObjectPosition(1) : _objectPos = _vector3
getObjectAngle(1) : _objectAng = _vector3
// setup ray
move camera 20
getCameraPosition()
move camera -20
_rayCollision = SC_rayCast(0,_cameraPos.x,_cameraPos.y,_cameraPos.z,_vector3.x,_vector3.y,_vector3.z,0)
// update collision
if _rayCollision
_rayHitPosition.x = SC_getStaticCollisionX() - _objectPos.x
_rayHitPosition.y = SC_getStaticCollisionY() - _objectPos.y
_rayHitPosition.z = SC_getStaticCollisionZ() - _objectPos.z
_faceHit = SC_GetFaceHit()
lock vertexdata for limb 1,0,0
vert = _faceHit * 3
_vector3.x = memblock float( 1 , (vert+0) * _memVertSize + 12 )
_vector3.y = memblock float( 1 , (vert+0) * _memVertSize + 16 )
_vector3.z = memblock float( 1 , (vert+0) * _memVertSize + 20 )
setObjectPosition( _dummyMesh, _objectPos )
setObjectAngle( _dummyMesh, _objectAng )
offset limb _dummyMesh, 0, _vector3.x, _vector3.y, _vector3.z
setLimbPositionToVector3( _dummyMesh, 0 ) : a = _vector3
_vector3.x = memblock float( 1 , (vert+1) * _memVertSize + 12 )
_vector3.y = memblock float( 1 , (vert+1) * _memVertSize + 16 )
_vector3.z = memblock float( 1 , (vert+1) * _memVertSize + 20 )
setObjectPosition( _dummyMesh, _objectPos )
setObjectAngle( _dummyMesh, _objectAng )
offset limb _dummyMesh, 0, _vector3.x, _vector3.y, _vector3.z
setLimbPositionToVector3( _dummyMesh, 0 ) : b = _vector3
_vector3.x = memblock float( 1 , (vert+2) * _memVertSize + 12 )
_vector3.y = memblock float( 1 , (vert+2) * _memVertSize + 16 )
_vector3.z = memblock float( 1 , (vert+2) * _memVertSize + 20 )
setObjectPosition( _dummyMesh, _objectPos )
setObjectAngle( _dummyMesh, _objectAng )
offset limb _dummyMesh, 0, _vector3.x, _vector3.y, _vector3.z
setLimbPositionToVector3( _dummyMesh, 0 ) : c = _vector3
unlock vertexdata
// work out barycentric coordinates (needs editing)
d# = 1.0 / ((b.y - c.y) * (a.x - c.x) + (c.x - b.x) * (a.y - c.y))
_vector3.x = ((b.y - c.y) * (_rayHitPosition.x - c.x) + (c.x - b.x) * (_rayHitPosition.y - c.y)) * d#
_vector3.y = ((c.y - a.y) * (_rayHitPosition.x - c.x) + (a.x - c.x) * (_rayHitPosition.y - c.y)) * d#
_vector3.z = 1 - _vector3.x - _vector3.y
// reverse coords depending on face (only quad)
if _faceHit
setPoint2( (_vector3.x*_texSize), _texSize-(_vector3.z*_texSize) )
else
setPoint2( _texSize-(_vector3.x*_texSize), (_vector3.z*_texSize) )
endif
_texCoordinates = _point2
// update image on mouse click
if mouseclick() = 1
set current bitmap 1
dot _texCoordinates.x,_texCoordinates.y, rgb(255,255,255)
get image 1,0,0,_texSize,_texSize,1
texture object 1,1
set current bitmap 0
endif
endif
// draw crosshair
dot _screenWidthHalf, _screenHeightHalf
text 0,0,"Press Mouse 1 to paint"
text 0,12,"Collision: "+str$(_rayCollision)
sync
loop
delete memblock 1
function setPoint2( x as float, y as float )
_point2.x = int(x) : _point2.y = int(y)
endfunction
function setVector3( x as float, y as float, z as float )
_vector3.x = x : _vector3.y = y : _vector3.z = z
endfunction
function setLimbPositionToVector3( id as integer, limb as integer )
setVector3( limb position x(id,limb), limb position y(id,limb), limb position z(id,limb) )
endfunction
function getCameraPosition()
_vector3.x = camera position x() : _vector3.y = camera position y() : _vector3.z = camera position z()
endfunction
function getCameraAngle()
_vector3.x = camera angle x() : _vector3.y = camera angle y() : _vector3.z = camera angle z()
endfunction
function setObjectPosition( id as integer, v as vector3 )
position object id, v.x, v.y, v.z
endfunction
function setObjectAngle( id as integer, v as vector3 )
rotate object id, v.x, v.y, v.z
endfunction
function getObjectPosition( id as integer )
_vector3.x = object position x(id) : _vector3.y = object position y(id) : _vector3.z = object position z(id)
endfunction
function getObjectAngle( id as integer )
_vector3.x = object angle x(id) : _vector3.y = object angle y(id) : _vector3.z = object angle z(id)
endfunction
Edit: After a little testing and so far I haven't found a bug, I can say this is twice as fast as Kira's method. I'm guessing all that matrix maths is slow.