diff --git a/CHANGELOG.md b/CHANGELOG.md
index 82dd02d..ee50496 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -65,6 +65,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- `persp`: add dirty flag to the structure, that when set means that at least one buffer was removed from the perspective manipulating the frame's hash table without updating the perspective's windows configuration.
+- `persp-maybe-kill-buffer`: set a perspective's dirty flag when removing buffers accessing the frame's hash table directly.
+- `persp-activate`: forget windows buffers which do not belong to the current perspective, hence updating the windows configuration; this is required since `persp-maybe-kill-buffer` no longer updates the perspectives windows configuration.
 - `persp-mode`: when enabling the mode, activate `persp-maybe-kill-buffer-adv`.
 - `persp-maybe-kill-buffer-adv`: due to `persp-maybe-kill-buffer` amendments, after calling `kill-buffer` force update the `current-buffer` to the current window's buffer.
 - `persp-maybe-kill-buffer`: remove buffers directly accessing the frame's hash table for performance reasons.
diff --git a/perspective.el b/perspective.el
index 39229cf..1b75845 100644
--- a/perspective.el
+++ b/perspective.el
@@ -166,7 +166,7 @@ After BODY is evaluated, frame parameters are reset to their original values."
 (cl-defstruct (perspective
                (:conc-name persp-)
                (:constructor make-persp-internal))
-  name buffers killed local-variables
+  name buffers killed dirty local-variables
   (last-switch-time (current-time))
   (created-time (current-time))
   (window-configuration (current-window-configuration))
@@ -753,19 +753,43 @@ If NORECORD is non-nil, do not update the
   (persp-update-modestring))
 
 (defun persp-activate (persp)
-  "Activate the perspective given by the persp struct PERSP."
+  "Activate the perspective given by the persp struct PERSP.
+
+If PERSP has a dirty flag set, some of its buffers may have been
+removed by directly manipulating the frame's hash table, but the
+perspective's windows configuration was not updated.  This means
+that the windows configuration may hold removed buffers that are
+pulled back into the perspective.  If this happens, buffers that
+do not belong to PERSP will be forgotten automatically."
   (check-persp persp)
   (persp-save)
   (set-frame-parameter nil 'persp--curr persp)
   (persp-reset-windows)
   (persp-set-local-variables (persp-local-variables persp))
   (setf (persp-buffers persp) (persp-reactivate-buffers (persp-buffers persp)))
-  (set-window-configuration (persp-window-configuration persp))
-  (when (marker-position (persp-point-marker persp))
-    (goto-char (persp-point-marker persp)))
+  ;; reset windows configuration
+  (let* ((dirty (persp-dirty persp))
+         windows-buffers-not-in-current-perspective
+         (buffers (and dirty (persp-current-buffers))))
+    (set-window-configuration (persp-window-configuration persp))
+    (when (marker-position (persp-point-marker persp))
+      (goto-char (persp-point-marker persp)))
+    ;; find windows buffers not in the current perspective
+    (when buffers
+      (walk-windows (lambda (window)
+                      (let ((buffer (window-buffer window)))
+                        (unless (memq buffer buffers)
+                          (push buffer windows-buffers-not-in-current-perspective)))))
+      ;; forget windows buffers not in the current perspective
+      (mapc (lambda (buffer)
+              ;; it's required to acknowledge the buffer to forget it
+              (persp-add-buffer buffer)
+              (persp-forget-buffer buffer))
+            windows-buffers-not-in-current-perspective)))
   (persp-update-modestring)
   ;; force update of `current-buffer'
   (set-buffer (window-buffer))
+  (setf (persp-dirty persp) nil)
   (run-hooks 'persp-activated-hook))
 
 (defun persp-switch-quick (char)
@@ -941,6 +965,7 @@ See also `persp-remove-buffer'."
                          ;; windows configuration.  A removed buffer
                          ;; will be pulled back by `persp-activate',
                          ;; if it's in the windows configuration.
+                         (setf (persp-dirty persp) t)
                          (setf (persp-buffers persp) other-buffers))
                      ;; Keep the buffer in this perspective.
                      (setq candidates-for-keeping t))))