diff --git a/internal/oci/runtime_vm.go b/internal/oci/runtime_vm.go
index 1ba8ffe6f9b..1fa0dcf9a0d 100644
--- a/internal/oci/runtime_vm.go
+++ b/internal/oci/runtime_vm.go
@@ -258,6 +258,12 @@ func (r *runtimeVM) StartContainer(c *Container) error {
 			if err1 := r.updateContainerStatus(c); err1 != nil {
 				logrus.Warningf("error updating container status %v", err1)
 			}
+
+			if c.state.Status == ContainerStateStopped {
+				if err1 = r.deleteContainer(c, true); err1 != nil {
+					logrus.WithError(cleanupErr).Infof("deleteContainer failed for container %s", c.ID())
+				}
+			}
 		} else {
 			logrus.Warningf("wait for %s returned: %v", c.ID(), err)
 		}