Javascript does not support operator overloading (i.e. specifying how "-" on something that isn't a number should work), you should call whatever functions your vectors provide for that (I can't find any documentation that cocos2d-js provides that API).
The NaN is because javascript is going "I don't know how to multiply these 2 things that aren't numbers, so that's Not A Number (NaN)"
velocityComputer: function(inVector, thisNVector) {
var prodVec = cc.pDot(inVector, thisNVector);
var retVec = cc.pSub( inVector, cc.pMult( cc.pCompMult( prodVec, inVector ), 2 ) );
cc.log(retVec);
// cc.log(blockNVec);
return retVec;
},