diff --git a/internal/config/nsmgr/nsmgr.go b/internal/config/nsmgr/nsmgr.go
index 31b87e91cd3..c3155bf4059 100644
--- a/internal/config/nsmgr/nsmgr.go
+++ b/internal/config/nsmgr/nsmgr.go
@@ -136,6 +136,11 @@ func (mgr *NamespaceManager) NewPodNamespaces(cfg *PodNamespacesConfig) ([]Names
 	for _, ns := range cfg.Namespaces {
 		ns, err := GetNamespace(ns.Path, ns.Type)
 		if err != nil {
+			for _, nsToClose := range returnedNamespaces {
+				if err2 := nsToClose.Remove(); err2 != nil {
+					logrus.Errorf("failed to remove namespace after failed to create: %v", err2)
+				}
+			}
 			return nil, err
 		}
 
diff --git a/internal/config/nsmgr/types.go b/internal/config/nsmgr/types.go
index e84c6353712..c7c7aaa08bd 100644
--- a/internal/config/nsmgr/types.go
+++ b/internal/config/nsmgr/types.go
@@ -116,10 +116,6 @@ func (n *namespace) Remove() error {
 // GetNamespace takes a path and type, checks if it is a namespace, and if so
 // returns an instance of the Namespace interface.
 func GetNamespace(nsPath string, nsType NSType) (Namespace, error) {
-	if err := nspkg.IsNSorErr(nsPath); err != nil {
-		return nil, err
-	}
-
 	ns, err := nspkg.GetNS(nsPath)
 	if err != nil {
 		return nil, err
diff --git a/internal/lib/container_server.go b/internal/lib/container_server.go
index 399ce89e6dd..5f0e9cc7cce 100644
--- a/internal/lib/container_server.go
+++ b/internal/lib/container_server.go
@@ -199,30 +199,30 @@ func (c *ContainerServer) LoadSandbox(ctx context.Context, id string) (retErr er
 	sb.SetSeccompProfilePath(spp)
 	sb.SetNamespaceOptions(&nsOpts)
 
+	defer func() {
+		if retErr != nil {
+			if err := sb.RemoveManagedNamespaces(); err != nil {
+				log.Warnf(ctx, "failed to remove namespaces: %v", err)
+			}
+		}
+	}()
 	// We add an NS only if we can load a permanent one.
 	// Otherwise, the sandbox will live in the host namespace.
-	netNsPath, err := configNsPath(&m, rspec.NetworkNamespace)
-	if err == nil {
-		if nsErr := sb.NetNsJoin(netNsPath); nsErr != nil {
-			return nsErr
-		}
-	}
-	ipcNsPath, err := configNsPath(&m, rspec.IPCNamespace)
-	if err == nil {
-		if nsErr := sb.IpcNsJoin(ipcNsPath); nsErr != nil {
-			return nsErr
-		}
-	}
-	utsNsPath, err := configNsPath(&m, rspec.UTSNamespace)
-	if err == nil {
-		if nsErr := sb.UtsNsJoin(utsNsPath); nsErr != nil {
-			return nsErr
-		}
-	}
-	userNsPath, err := configNsPath(&m, rspec.UserNamespace)
-	if err == nil {
-		if nsErr := sb.UserNsJoin(userNsPath); nsErr != nil {
-			return nsErr
+	namespacesToJoin := []struct {
+		rspecNS  rspec.LinuxNamespaceType
+		joinFunc func(string) error
+	}{
+		{rspecNS: rspec.NetworkNamespace, joinFunc: sb.NetNsJoin},
+		{rspecNS: rspec.IPCNamespace, joinFunc: sb.IpcNsJoin},
+		{rspecNS: rspec.UTSNamespace, joinFunc: sb.UtsNsJoin},
+		{rspecNS: rspec.UserNamespace, joinFunc: sb.UserNsJoin},
+	}
+	for _, namespaceToJoin := range namespacesToJoin {
+		path, err := configNsPath(&m, namespaceToJoin.rspecNS)
+		if err == nil {
+			if nsErr := namespaceToJoin.joinFunc(path); err != nil {
+				return nsErr
+			}
 		}
 	}
 
@@ -304,28 +304,6 @@ func (c *ContainerServer) LoadSandbox(ctx context.Context, id string) (retErr er
 		return err
 	}
 
-	// We add an NS only if we can load a permanent one.
-	// Otherwise, the sandbox will live in the host namespace.
-	if wasSpoofed {
-		namespacesToJoin := []struct {
-			rspecNS  rspec.LinuxNamespaceType
-			joinFunc func(string) error
-		}{
-			{rspecNS: rspec.NetworkNamespace, joinFunc: sb.NetNsJoin},
-			{rspecNS: rspec.IPCNamespace, joinFunc: sb.IpcNsJoin},
-			{rspecNS: rspec.UTSNamespace, joinFunc: sb.UtsNsJoin},
-			{rspecNS: rspec.UserNamespace, joinFunc: sb.UserNsJoin},
-		}
-		for _, namespaceToJoin := range namespacesToJoin {
-			path, err := configNsPath(&m, namespaceToJoin.rspecNS)
-			if err == nil {
-				if nsErr := namespaceToJoin.joinFunc(path); err != nil {
-					return nsErr
-				}
-			}
-		}
-	}
-
 	sb.SetCreated()
 	if err := label.ReserveLabel(processLabel); err != nil {
 		return err