diff --git a/gtsam/linear/VectorValues.cpp b/gtsam/linear/VectorValues.cpp
index 4edca2521..96bafcd16 100644
--- a/gtsam/linear/VectorValues.cpp
+++ b/gtsam/linear/VectorValues.cpp
@@ -168,6 +168,9 @@ VectorValues& VectorValues::operator=(const Permuted<VectorValues>& rhs) {
       if(l.rows() != r.rows())
         throw std::invalid_argument("VectorValues assignment from Permuted<VectorValues> requires pre-allocation, see documentation.");
       l = r;
+    } else {
+      if(rhs.container().exists(rhs.permutation()[j]))
+        throw std::invalid_argument("VectorValues assignment from Permuted<VectorValues> requires pre-allocation, see documentation.");
     }
   }
   return *this;