Personally, I needed something with a little longer range than the default when I needed this. If that's the case for you too, I pretty much copied getMouseOver(), changing a few variables and names to get what I needed. Here's the original:
public void getMouseOver(float par1)
{
if (this.mc.renderViewEntity != null)
{
if (this.mc.theWorld != null)
{
this.mc.pointedEntity = null;
double d0 = (double)this.mc.playerController.getBlockReachDistance();
this.mc.objectMouseOver = this.mc.renderViewEntity.rayTrace(d0, par1);
double d1 = d0;
Vec3 vec3 = this.mc.renderViewEntity.getPosition(par1);
if (this.mc.playerController.extendedReach())
{
d0 = 6.0D;
d1 = 6.0D;
}
else
{
if (d0 > 3.0D)
{
d1 = 3.0D;
}
d0 = d1;
}
if (this.mc.objectMouseOver != null)
{
d1 = this.mc.objectMouseOver.hitVec.distanceTo(vec3);
}
Vec3 vec31 = this.mc.renderViewEntity.getLook(par1);
Vec3 vec32 = vec3.addVector(vec31.xCoord * d0, vec31.yCoord * d0, vec31.zCoord * d0);
this.pointedEntity = null;
Vec3 vec33 = null;
float f1 = 1.0F;
List list = this.mc.theWorld.getEntitiesWithinAABBExcludingEntity(this.mc.renderViewEntity, this.mc.renderViewEntity.boundingBox.addCoord(vec31.xCoord * d0, vec31.yCoord * d0, vec31.zCoord * d0).expand((double)f1, (double)f1, (double)f1));
double d2 = d1;
for (int i = 0; i < list.size(); ++i)
{
Entity entity = (Entity)list.get(i);
if (entity.canBeCollidedWith())
{
float f2 = entity.getCollisionBorderSize();
AxisAlignedBB axisalignedbb = entity.boundingBox.expand((double)f2, (double)f2, (double)f2);
MovingObjectPosition movingobjectposition = axisalignedbb.calculateIntercept(vec3, vec32);
if (axisalignedbb.isVecInside(vec3))
{
if (0.0D < d2 || d2 == 0.0D)
{
this.pointedEntity = entity;
vec33 = movingobjectposition == null ? vec3 : movingobjectposition.hitVec;
d2 = 0.0D;
}
}
else if (movingobjectposition != null)
{
double d3 = vec3.distanceTo(movingobjectposition.hitVec);
if (d3 < d2 || d2 == 0.0D)
{
if (entity == this.mc.renderViewEntity.ridingEntity && !entity.canRiderInteract())
{
if (d2 == 0.0D)
{
this.pointedEntity = entity;
vec33 = movingobjectposition.hitVec;
}
}
else
{
this.pointedEntity = entity;
vec33 = movingobjectposition.hitVec;
d2 = d3;
}
}
}
}
}
if (this.pointedEntity != null && (d2 < d1 || this.mc.objectMouseOver == null))
{
this.mc.objectMouseOver = new MovingObjectPosition(this.pointedEntity, vec33);
if (this.pointedEntity instanceof EntityLivingBase || this.pointedEntity instanceof EntityItemFrame)
{
this.mc.pointedEntity = this.pointedEntity;
}
}
}
I put this all in it's own method, making float par1 an argument (which has something to do with ticks, put 1.0F) and how far away you want to look, which is double d0. I subtracted the if-else loop that changes d0 and d1 according to player reach (4.5 survival, 6.0 creative), and used the mc object in my class. Here's my modified version, which returns the EntityLivingBase that the cursor is on:
private EntityLivingBase getTarget(float par1, double distance)
{
Minecraft mc = Minecraft.getMinecraft();
Entity pointedEntity;
double d0 = distance;
MovingObjectPosition omo = mc.renderViewEntity.rayTrace(d0, par1);
double d1 = d0;
Vec3 vec3 = mc.renderViewEntity.getPosition(par1);
Vec3 vec31 = mc.renderViewEntity.getLook(par1);
Vec3 vec32 = vec3.addVector(vec31.xCoord * d0, vec31.yCoord * d0, vec31.zCoord * d0);
pointedEntity = null;
Vec3 vec33 = null;
float f1 = 1.0F;
List list = mc.theWorld.getEntitiesWithinAABBExcludingEntity(mc.renderViewEntity, mc.renderViewEntity.boundingBox.addCoord(vec31.xCoord * d0, vec31.yCoord * d0, vec31.zCoord * d0).expand((double)f1, (double)f1, (double)f1));
double d2 = d1;
for (int i = 0; i < list.size(); ++i)
{
Entity entity = (Entity)list.get(i);
if (entity.canBeCollidedWith())
{
float f2 = entity.getCollisionBorderSize();
AxisAlignedBB axisalignedbb = entity.boundingBox.expand((double)f2, (double)f2, (double)f2);
MovingObjectPosition movingobjectposition = axisalignedbb.calculateIntercept(vec3, vec32);
if (axisalignedbb.isVecInside(vec3))
{
if (0.0D < d2 || d2 == 0.0D)
{
pointedEntity = entity;
vec33 = movingobjectposition == null ? vec3 : movingobjectposition.hitVec;
d2 = 0.0D;
}
}
else if (movingobjectposition != null)
{
double d3 = vec3.distanceTo(movingobjectposition.hitVec);
if (d3 < d2 || d2 == 0.0D)
{
if (entity == mc.renderViewEntity.ridingEntity && !entity.canRiderInteract())
{
if (d2 == 0.0D)
{
pointedEntity = entity;
vec33 = movingobjectposition.hitVec;
}
}
else
{
pointedEntity = entity;
vec33 = movingobjectposition.hitVec;
d2 = d3;
}
}
}
}
}
if (pointedEntity != null && (d2 < d1 || omo == null))
{
omo = new MovingObjectPosition(pointedEntity, vec33);
if (pointedEntity instanceof EntityLivingBase || pointedEntity instanceof EntityItemFrame)
{
mc.pointedEntity = pointedEntity;
}
}
if (omo != null)
{
if (omo.typeOfHit == MovingObjectType.ENTITY)
{
if(omo.entityHit instanceof EntityLivingBase)
{
return (EntityLivingBase)omo.entityHit;
}
}
}
return null;
}
Hope this helps, and good luck with your mod.