I'm trying to do drag to rotate sort of thing, so far the code is fine. However when starting to drag (left clicking first), the object rotate to a random direction first which looks odd.

here is my code :

public Camera cam;
public float smoothAngle, dragPower = .3f;
public bool isRadial = true;
float angle, angleOld, angleDelta, angleRef;
Vector2 mousePosOld, mousePosDelta = Vector2.zero;

void Update()
{
  Vector2 mousePos = cam.ScreenToWorldPoint(Input.mousePosition);
  if (Input.GetMouseButton(0))
  {
    mousePosDelta = mousePos - mousePosOld; 
    float rotX = (mousePosDelta.magnitude / Time.deltaTime) 
                  * dragPower 
                  * -Mathf.Sign(mousePos.y) 
                  * Mathf.Sign(Vector2.Dot(Vector2.right, mousePosDelta));
      
    transform.Rotate(Vector3.forward, rotX);
    
    mousePosOld = mousePos;
    angleDelta = Mathf.DeltaAngle(angleOld, transform.localEulerAngles.z);
    angleOld = transform.localEulerAngles.z;
  }
  else
  {
    transform.RotateAround(transform.position, transform.forward, angleDelta);
    angleDelta = Mathf.SmoothDampAngle(angleDelta, 0, ref angleRef, smoothAngle);
  }
}

this is the effect :