# Problem detecting outer edges properly when deforming mesh dynamically

I've been using this edge code (https://answers.unity.com/questions/1019436/get-outeredge-vertices-c.html) for detecting outer edges.

     public struct Edge
{
public int v1;
public int v2;
public int triangleIndex;
public Edge(int aV1, int aV2, int aIndex)
{
v1 = aV1;
v2 = aV2;
triangleIndex = aIndex;
}
}

public static List<Edge> GetEdges(int[] aIndices)
{
List<Edge> result = new List<Edge>();
for (int i = 0; i < aIndices.Length; i += 3)
{
int v1 = aIndices[i];
int v2 = aIndices[i + 1];
int v3 = aIndices[i + 2];
}
return result;
}

public static List<Edge> FindBoundary(this List<Edge> aEdges)
{
List<Edge> result = new List<Edge>(aEdges);
for (int i = result.Count-1; i > 0; i--)
{
for (int n = i - 1; n >= 0; n--)
{
if (result[i].v1 == result[n].v2 && result[i].v2 == result[n].v1)
{
// shared edge so remove both
result.RemoveAt(i);
result.RemoveAt(n);
i--;
break;
}
}
}
return result;
}



But I've run into a problem where it doesn't register outer edge vertices properly when used with a basic subdivision function.

GIF of how the subdivision works:

    private void SubdivideEdge(Mesh mesh, Edge targetEdge, List<Vector3> verts, List<int> tris, List<Vector2> uvs, List<Vector3> normals)
{
//Edges used to find triangles for deletion and used for subdivision.
SelectedEdges = new List<Edge>();

//Find duplicate edge. You will typically end up with 2 edges with the same vertices.
//Use these 2 edges to find the 2 triangles.
foreach (var edge in EdgeList)
{
if (((edge.v1 == targetEdge.v1 && edge.v2 == targetEdge.v2) || (edge.v1 == targetEdge.v2 && edge.v2 == targetEdge.v1))
&& edge.triangleIndex != targetEdge.triangleIndex)
{
break;
}
}

var centerVertIndex = 0;
var newVertPos = verts[targetEdge.v1] + (verts[targetEdge.v2] - verts[targetEdge.v1]) / 2; //midpoint of target edge

//add the new vertex, uv, and normal to the list
uvs.Add(uvs[targetEdge.v1] + (uvs[targetEdge.v2] - uvs[targetEdge.v1]) / 2);
centerVertIndex = verts.Count - 1;

//add four new triangles by using the indices of the two edge vertices as well as the 3rd index of the triangle associated with the edge
for (int i = 0; i < SelectedEdges.Count; i++)
{
var triIndex = SelectedEdges[i].triangleIndex;

var thirdVertIndex = 0;

if (tris[triIndex] != targetEdge.v1 && tris[triIndex] != targetEdge.v2)
{
thirdVertIndex = tris[triIndex];
}
else if (tris[triIndex + 1] != targetEdge.v1 && tris[triIndex + 1] != targetEdge.v2)
{
thirdVertIndex = tris[triIndex + 1];
}
else if (tris[triIndex + 2] != targetEdge.v1 && tris[triIndex + 2] != targetEdge.v2)
{
thirdVertIndex = tris[triIndex + 2];
}

var vertIndexA = centerVertIndex;
var vertIndexB = SelectedEdges[i].v1;
var vertIndexC = thirdVertIndex;
var vertIndexD = SelectedEdges[i].v2;

var triPointA = verts[vertIndexA];
var triPointB = verts[vertIndexB];
var triPointC = verts[vertIndexC];
var triPointD = verts[vertIndexD];

var edgeBA = triPointB - triPointA;
var edgeAC = triPointA - triPointC;
var edgeDA = triPointD - triPointA;

//Use Vector3.Cross to find the correct winding order.
var sign = Vector3.Cross(edgeBA, edgeAC).z;
var sign2 = Vector3.Cross(edgeDA, edgeAC).z;

if (sign > 0)
{
}
else if (sign < 0)
{
}

if (sign2 > 0)
{
}
else if (sign2 < 0)
{
}

}

//update all of the triangle indices for each edge
var previousTriIndex = 0;
for (int i = 0; i < SelectedEdges.Count; i++)
{
//Debug.Log(SelectedEdges[i].triangleIndex + " INDEX");
previousTriIndex = SelectedEdges[i].triangleIndex;
tris.RemoveAt(SelectedEdges[i].triangleIndex);
tris.RemoveAt(SelectedEdges[i].triangleIndex);
tris.RemoveAt(SelectedEdges[i].triangleIndex);

foreach (var edge in EdgeList)
{
if (previousTriIndex < edge.triangleIndex)
{
edge.triangleIndex -= 3;
}
}

}

mesh.SetVertices(Vertices);
mesh.SetUVs(0, UVs);
mesh.SetNormals(Normals);
mesh.SetTriangles(Triangles, 0);
}


This is the result: https://i.gyazo.com/7a86e74183ea8285d207927fc15c1301.gif

You can see that as the vertices get pushed, it recognizes the inner mesh vertices as outer edge vertices (in cyan color) for some strange reason. This glitch has caused tearing to happen within the mesh I'm trying to deform.

https://i.gyazo.com/4db38e52406443251a7789f72af6bf95.gif

I try the same subdivision code on a mesh that doesn't move the vertices and it works without the issues shown above. Since its caused by movement of the vertices, I thought it was a position problem but the edge script only uses vertex index and triangle index so I'm not sure why when vertices are moving that this causes the problem.

• Where do you remove/modify the split edges from the edge list and insert the new edges you've created during subdivision? Oct 27, 2021 at 16:46
• @DMGregory I do it after the subdivision is applied. I call GetEdges and FindBoundary to create a new list of updated edges. Oct 27, 2021 at 17:57