My answer is simple ... this problem is hard in an arbitrary coordinate system, so Change it to something that makes the problem easy. The Matrix class in xna has a CreateLookAt function which can be used to create a useful transformation on all verticies.
The following example is not optimized, it is written only for understanding of the solution. The Exceptions and their corresponding if statements can all be removed, along with a few vector transformations.
public static bool CheckColision(Vector3 t1a, Vector3 t1b, Vector3 t1c, Vector3 t2a, Vector3 t2b, Vector3 t2c)
{//rotates each edge of the first triangle to the Z axis and checks the second triangle against it then repeats with the second one against the first, and lastly checks to see if all points of the second triangle are on the same side as the first
if(! CheckColisionLookAt(t1a, t1b, t1c, t2a, t2b, t2c))
return false;
if (!CheckColisionLookAt(t1b, t1c, t1a, t2a, t2b, t2c))
return false;
if (!CheckColisionLookAt(t1c, t1a, t1b, t2a, t2b, t2c))
return false;
if (!CheckColisionLookAt(t2a, t2b, t2c, t1a, t1b, t1c))
return false;
if (!CheckColisionLookAt(t2b, t2c, t2a, t1a, t1b, t1c))
return false;
if (!CheckColisionLookAt(t2c, t2a, t2b, t1a, t1b, t1c))
return false;
return CheckColisionAllOnOneSide(t1a, t1b, t1c, t2a, t2b, t2c);
}
public static bool CheckColisionAllOnOneSide(Vector3 t1a, Vector3 t1b, Vector3 t1c, Vector3 t2a, Vector3 t2b, Vector3 t2c)
{//simply performs a transformation to check if all points on one triangle are on the same side of the other triangle
Matrix m = Matrix.CreateLookAt(t1a, t1b, t1c - t1a);
t2a = Vector3.Transform(t2a, m);
t2b = Vector3.Transform(t2b, m);
t2c = Vector3.Transform(t2c, m);
if (t2a.X < 0 && t2b.X < 0 && t2c.X < 0)
return false;
if (0 < t2a.X && 0 < t2b.X && 0 < t2c.X)
return false;
return true;
}
public static bool CheckColisionLookAt(Vector3 t1a, Vector3 t1b, Vector3 t1c, Vector3 t2a, Vector3 t2b, Vector3 t2c)
{//performs a transformation and checks if all points of the one triangle are under the other triangle after the transformation
Matrix m = Matrix.CreateLookAt(t1a, t1b, t1c - t1a);
t1a = Vector3.Transform(t1a, m);// (0, 0, 0)
if ( ZERRO < Math.Abs(t1a.X)|| ZERRO < Math.Abs(t1a.Y) || ZERRO < Math.Abs(t1a.Z))
throw new Exception();
t1b = Vector3.Transform(t1b, m);// (0, 0, maxZ)
if (ZERRO < Math.Abs(t1a.X) || ZERRO < Math.Abs(t1a.Y))
throw new Exception();
t1c = Vector3.Transform(t1c, m);// (0, maxY, someZ)
if (ZERRO < Math.Abs(t1a.X))
throw new Exception();
t2a = Vector3.Transform(t2a, m);
t2b = Vector3.Transform(t2b, m);
t2c = Vector3.Transform(t2c, m);
if (t2a.Y < 0 && t2b.Y < 0 && t2c.Y < 0)
return false;
return true;
}