Object pooling in Multiplayer Game. Physics not synchronizing well

Question

I implemented object pooling for my multiplayer project. Following Unity Manual I used custom spawn handler for shooting bullets. Bullets game object has rigidbody so selected Sync Rigibody 3D in Network Transfrom. My pool size is 5.

While shooting when all 5 projectile is shooted then they have to have to come back to initial position one by one that is the concept of pooling.

When projectile moving back they just instantly go back to initial position on host but on client their position are lerping and can easily be seen that we are reusing projectile.

How will I fix this ? I've added GIF for better understanding

Code For SpawnManager where I'am using Custom SpawnHandler

public class SpawnManager : MonoBehaviour {

public int m_ObjectPoolSize = 10;
public GameObject m_Prefab;
public GameObject[] m_Pool;

private Queue<GameObject> objectPool;
public NetworkHash128 assetId { get; set; }

public delegate GameObject SpawnDelegate(Vector3 position, NetworkHash128 assetId);
public delegate void UnSpawnDelegate(GameObject spawned);

void Start()
{
    assetId = m_Prefab.GetComponent<NetworkIdentity>().assetId;

    objectPool = new Queue<GameObject>();
    for (int i = 0; i < m_ObjectPoolSize; ++i)
    {
        GameObject obj = (GameObject)Instantiate(m_Prefab, Vector3.zero, Quaternion.identity);
        obj.SetActive(false);
        objectPool.Enqueue(obj);
    }

    ClientScene.RegisterSpawnHandler(assetId, SpawnObject, UnSpawnObject);
}

public GameObject GetFromPool(Vector3 position)
{
    GameObject obj = objectPool.Dequeue();
    obj.transform.position = position;
    obj.SetActive(true);
    objectPool.Enqueue(obj);

    return obj;
}

public GameObject SpawnObject(Vector3 position, NetworkHash128 assetId)
{
    return GetFromPool(position);
}
// Not calling this function yet
public void UnSpawnObject(GameObject spawned)
{
    Debug.Log("Re-pooling GameObject " + spawned.name);
    spawned.SetActive(false);
}

}

And this is the code where I am calling these function.

public class PlayerTest : NetworkBehaviour {

SpawnManager spawnManager;

private void Start()
{
    spawnManager = GameObject.Find("Spawn Manager").GetComponent<SpawnManager>();
}

private void Update() {

    if (!isLocalPlayer) return;

    if(Input.GetKeyDown(KeyCode.Space)) {

        CmdFire();
    }
}

[Command]
void CmdFire() {

    // Set up prefab on server
    var prefab = spawnManager.GetFromPool(transform.position + transform.forward);
    prefab.GetComponent<Rigidbody>().velocity = transform.forward * 4;

    // spawn prefab on client and server
    NetworkServer.Spawn(prefab, spawnManager.assetId);
}