Why are my cached (in an array) gameobjects disappearing?

Question

First, some context: I am developing a chunk loading system for unity. I load all the chunks as prefabs from disk to an array as a cache. When I need to load a chunk, I instantiate it from the cache into the world.
The problem is that when deleting gameobjects from the world (using the destroy method) to unload them, it also destroys the cached copy meaning that I can't instantiate it back later.
I've done some of my own research and I have found that some time after I initially cached the gameobjects and instantiated them into the scene for the first time, the hash codes of both the cached and world gameobjects change to become the same thing. This occurs while the editor isn't playing and no other scripts are running.

Here is some code:

 void ImportChunks ()
{
    //Reset All Chunks
    //prefabs = new List<GameObject>();
    prefabs.Clear ();

    prefSorted = new List<List<List<GameObject>>> ();

    //Fill the list
    prefSorted = Enumerable.Range (0, Layers).Select (l =>
        Enumerable.Range (0, GridXSize).Select (h =>
            Enumerable.Range (0, GridYSize).Select (w => 
                new GameObject ()).ToList ()).ToList ()).ToList (); 

    Debug.Log (prefSorted.Count + " layers");
    Debug.Log (prefSorted [0].Count + " x");
    Debug.Log (prefSorted [0] [0].Count + " y");

    //Load Chunks From Disk
    string[] pref = AssetDatabase.FindAssets ("@", new string[]{ PrefabDir });
    int d = 0;
    foreach (string prefa in pref)
    {
        //Load chunk from disk
        GameObject n = ((GameObject)AssetDatabase.LoadAssetAtPath (AssetDatabase.GUIDToAssetPath (prefa), typeof(GameObject)));

        //Put it into the cache array
        int[] pos = GetPos (n.name);
        prefSorted [pos [0]] [pos [1]] [pos [2]] = n;

        d++;
    }

    //Add all of our children to the loaded array
    HashSet<GameObject> children = new HashSet<GameObject> ();

    for (int i = 0; i < this.transform.childCount; i++)
    {
        children.Add (this.transform.GetChild (i).gameObject);
    }

    loaded = children;
}

public GameObject LoadChunk (int l, int x, int y)
{
    //We have an array containing a reference to all the currently loaded chunks.
    //Here we find the chunk which is already at the coordinates where we want to load the new chunk.
    //BTW naming convention is x-y@l-lv
    GameObject old = loaded.FirstOrDefault (i => i.name.Split ("@" [0]) [0] == x + "-" + y);
    if (old != null)
    {
        if (old.name.Split ("@" [0]) [1].Split ("-" [0]) [0] == l.ToString ())
            return null;

        //Destroy the object and remove it from the array of REFERENCED objects
        Destroy (old);
        loaded.Remove (old);
    }

    //Make the new chunk
    //prefSorted is a cache of all chunks so they can be instantiated
    GameObject newChunk = Instantiate (prefSorted [l] [x] [y]);

    newChunk.transform.SetParent (this.transform);
    loaded.Add (newChunk);
    newChunk.transform.localPosition = new Vector3 (x * GridScale.x, 0, y * GridScale.y);

    return newChunk;
}

Thanks for everybody's help in advance. If you have any questions, leave a comment and I'll update the post. Thanks,
-Thomas