Keep in mind that pt1 and pt2 define the line. So the point returned may not be between pt1 and pt2.

Ogre::Vector3 nearestPoint(Ogre::Vector3 pt1, Ogre::Vector3 pt2, Ogre::Vector3 testPoint) 
{ 
   // Find the point on a line defined by pt1 and pt2 that 
   // is nearest to a given point tp 
   //       tp 
   //       /| 
   //    A / | 
   //     /  | 
   //    /   | 
   //  pt1---o--------------pt2 
   //     B'      B 

    // Get the vectors between the points 
   Ogre::Vector3 A = testPoint - pt1; 
   Ogre::Vector3 B = pt2 - pt1; 

   // Find the cos of the angle between the vectors 
   float cosTheta = A.dotProduct(B) / (A.length() * B.length()); 
   // Use that to calculate the length of B' 
   float BPrimeLength = A.length() * cosTheta; 
   // Find the ratio of the length of B' and B 
   float scale = BPrimeLength / B.length(); 
   // Scale B by that ratio 
   B *= scale; 
   // Translate p1 by B, this puts it at o 
   Ogre::Vector3 C = pt1 + B; 

   return C; 
}

A smaller and faster version:

inline Ogre::Vector3 nearestPoint(const Ogre::Vector3 &pt1, const Ogre::Vector3 &pt2, const Ogre::Vector3 &testPoint) 
{
    const Ogre::Vector3 A = testPoint - pt1;
    const Ogre::Vector3 u = (pt2-pt1).normalisedCopy();
 
    return pt1 + (A.dotProduct(u)) * u;
};

If you wish to find the closest point that exists between the 2 points in the line

//Give the spot on the line, thats closest to testPoint
inline Ogre::Vector3 nearestPointInBetween(const Ogre::Vector3 &pt1, const Ogre::Vector3 &pt2, const Ogre::Vector3 &testPoint) 
{
    //probably not most efficient(?)
    const Ogre::Vector3 result = nearestPoint(pt1,pt2,testPoint);
    const Ogre::Real lineLength = pt1.squaredDistance(pt2);

    const Ogre::Real p1R = pt1.squaredDistance(result);
    const Ogre::Real p2R = pt2.squaredDistance(result);
    //R                R p1 R p2 R                R
    if ( p1R > lineLength )
    {
        if ( p2R > p1R )
            return pt1; //pt 1 is closer to result
        else
            return pt2;
    }
    else if ( p2R > lineLength )
        return pt1;

    return result;
};