diff --git a/packages/block-library/src/image/index.php b/packages/block-library/src/image/index.php
index f29202fe63d195..c465677a986e05 100644
--- a/packages/block-library/src/image/index.php
+++ b/packages/block-library/src/image/index.php
@@ -235,6 +235,7 @@ function block_core_image_render_lightbox( $block_content, $block ) {
 	$button =
 		$img[0]
 		. '<button
+			class="lightbox-trigger"
 			type="button"
 			aria-haspopup="dialog"
 			aria-label="' . esc_attr( $aria_label ) . '"
@@ -243,7 +244,7 @@ function block_core_image_render_lightbox( $block_content, $block ) {
 			data-wp-style--top="context.core.image.imageButtonTop"
 			style="background: #000"
 		>
-			<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="#FFFFFF">
+			<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" aria-hidden="true" focusable="false">
 				<Path stroke="#FFFFFF" d="M6 4a2 2 0 0 0-2 2v3h1.5V6a.5.5 0 0 1 .5-.5h3V4H6Zm3 14.5H6a.5.5 0 0 1-.5-.5v-3H4v3a2 2 0 0 0 2 2h3v-1.5Zm6 1.5v-1.5h3a.5.5 0 0 0 .5-.5v-3H20v3a2 2 0 0 1-2 2h-3Zm3-16a2 2 0 0 1 2 2v3h-1.5V6a.5.5 0 0 0-.5-.5h-3V4h3Z" />
 			</svg>
 		</button>';
@@ -319,12 +320,13 @@ function block_core_image_render_lightbox( $block_content, $block ) {
             data-wp-on--touchmove="actions.core.image.handleTouchMove"
             data-wp-on--touchend="actions.core.image.handleTouchEnd"
             data-wp-on--click="actions.core.image.hideLightbox"
+            tabindex="-1"
             >
                 <button type="button" aria-label="$close_button_label" style="fill: $close_button_color" class="close-button" data-wp-on--click="actions.core.image.hideLightbox">
                     $close_button_icon
                 </button>
                 <div class="lightbox-image-container">$initial_image_content</div>
-				<div class="lightbox-image-container">$enlarged_image_content</div>
+                <div class="lightbox-image-container">$enlarged_image_content</div>
                 <div class="scrim" style="background-color: $background_color" aria-hidden="true"></div>
         </div>
 HTML;
diff --git a/packages/block-library/src/image/view.js b/packages/block-library/src/image/view.js
index 30d1259637e3d9..331c0e79c731fc 100644
--- a/packages/block-library/src/image/view.js
+++ b/packages/block-library/src/image/view.js
@@ -135,7 +135,7 @@ store(
 							false
 						);
 					},
-					hideLightbox: async ( { context, event } ) => {
+					hideLightbox: async ( { context } ) => {
 						context.core.image.hideAnimationEnabled = true;
 						if ( context.core.image.lightboxEnabled ) {
 							// We want to wait until the close animation is completed
@@ -149,19 +149,15 @@ store(
 									'scroll',
 									scrollCallback
 								);
+								// If we don't delay before changing the focus,
+								// the focus ring will appear on Firefox before
+								// the image has finished animating, which looks broken.
+								context.core.image.lightboxTriggerRef.focus( {
+									preventScroll: true,
+								} );
 							}, 450 );
 
 							context.core.image.lightboxEnabled = false;
-
-							// We want to avoid drawing attention to the button
-							// after the lightbox closes for mouse and touch users.
-							// Note that the `event.pointerType` property returns
-							// as an empty string if a keyboard fired the event.
-							if ( event.pointerType === '' ) {
-								context.core.image.lastFocusedElement.focus( {
-									preventScroll: true,
-								} );
-							}
 						}
 					},
 					handleKeydown: ( { context, actions, event } ) => {
@@ -266,6 +262,10 @@ store(
 				image: {
 					initOriginImage: ( { context, ref } ) => {
 						context.core.image.imageRef = ref;
+						context.core.image.lightboxTriggerRef =
+							ref.parentElement.querySelector(
+								'.lightbox-trigger'
+							);
 						if ( ref.complete ) {
 							context.core.image.imageLoaded = true;
 							context.core.image.imageCurrentSrc = ref.currentSrc;
@@ -282,14 +282,8 @@ store(
 									focusableElements.length - 1
 								];
 
-							// We want to avoid drawing unnecessary attention to the close
-							// button for mouse and touch users. Note that even if opening
-							// the lightbox via keyboard, the event fired is of type
-							// `pointerEvent`, so we need to rely on the `event.pointerType`
-							// property, which returns an empty string for keyboard events.
-							if ( context.core.image.pointerType === '' ) {
-								ref.querySelector( '.close-button' ).focus();
-							}
+							// Move focus to the dialog when opening it.
+							ref.focus();
 						}
 					},
 					setButtonStyles: ( { context, ref } ) => {