diff --git a/assets/src/stories-editor/components/higher-order/with-keyboard-navigation-handler.js b/assets/src/stories-editor/components/higher-order/with-keyboard-navigation-handler.js
new file mode 100644
index 00000000000..99469160335
--- /dev/null
+++ b/assets/src/stories-editor/components/higher-order/with-keyboard-navigation-handler.js
@@ -0,0 +1,118 @@
+/**
+ * WordPress dependencies
+ */
+import { withDispatch } from '@wordpress/data';
+import { createHigherOrderComponent } from '@wordpress/compose';
+import { UP, DOWN, RIGHT, LEFT } from '@wordpress/keycodes';
+import { KeyboardShortcuts } from '@wordpress/components';
+/**
+ * Internal dependencies
+ */
+import { ALLOWED_CHILD_BLOCKS, ALLOWED_MOVABLE_BLOCKS } from '../../constants';
+
+const applyWithDispatch = withDispatch( ( dispatch, props, { select } ) => {
+	const {	isReordering } = select( 'amp/story' );
+	const { getSelectedBlock } = select( 'core/block-editor' );
+	const { updateBlockAttributes, removeBlock } = dispatch( 'core/block-editor' );
+	const selectedBlock = getSelectedBlock();
+
+	const onMoveBlock = ( event ) => {
+		const { keyCode, target } = event;
+		const { classList } = target;
+
+		if ( ! selectedBlock ) {
+			return;
+		}
+
+		if ( classList.contains( 'editor-rich-text__editable' ) && classList.contains( 'is-selected' ) ) {
+			return;
+		}
+
+		let top = 0;
+		let left = 0;
+		switch ( keyCode ) {
+			case UP:
+				top = -1;
+				break;
+			case DOWN:
+				top = 1;
+				break;
+			case RIGHT:
+				left = 1;
+				break;
+			case LEFT:
+				left = -1;
+				break;
+			default:
+				break;
+		}
+
+		if ( ALLOWED_MOVABLE_BLOCKS.includes( selectedBlock.name ) && ( left || top ) ) {
+			event.preventDefault();
+			const newPositionTop = selectedBlock.attributes.positionTop + top;
+			const newPositionLeft = selectedBlock.attributes.positionLeft + left;
+			updateBlockAttributes( selectedBlock.clientId, {
+				positionTop: newPositionTop,
+				positionLeft: newPositionLeft,
+			} );
+		}
+	};
+
+	const deleteSelectedBlocks = ( event ) => {
+		const { target } = event;
+		const { classList } = target;
+		if ( ! selectedBlock ) {
+			return;
+		}
+		if ( classList.contains( 'editor-rich-text__editable' ) && classList.contains( 'is-selected' ) ) {
+			return;
+		}
+		event.preventDefault();
+		removeBlock( selectedBlock.clientId );
+	};
+
+	return {
+		isReordering,
+		onMoveBlock,
+		deleteSelectedBlocks,
+	};
+} );
+
+/**
+ * Higher-order component that adds right click handler to each inner block.
+ *
+ * @return {Function} Higher-order component.
+ */
+export default createHigherOrderComponent(
+	( BlockEdit ) => {
+		return applyWithDispatch( ( props ) => {
+			const { name, onMoveBlock, isReordering, deleteSelectedBlocks } = props;
+			const isPageBlock = 'amp/amp-story-page' === name;
+			// Add for page block and inner blocks.
+			if ( ! isPageBlock && ! ALLOWED_CHILD_BLOCKS.includes( name ) ) {
+				return <BlockEdit { ...props } />;
+			}
+
+			// Not relevant for reordering.
+			if ( isReordering() ) {
+				return <BlockEdit { ...props } />;
+			}
+
+			const shortcuts = {
+				up: onMoveBlock,
+				right: onMoveBlock,
+				down: onMoveBlock,
+				left: onMoveBlock,
+				backspace: deleteSelectedBlocks,
+				del: deleteSelectedBlocks,
+			};
+
+			return (
+				<KeyboardShortcuts shortcuts={ shortcuts } event="keyup">
+					<BlockEdit { ...props } />
+				</KeyboardShortcuts>
+			);
+		} );
+	},
+	'withKeyboardNavigationHandler'
+);
diff --git a/assets/src/stories-editor/components/index.js b/assets/src/stories-editor/components/index.js
index d6744a89c32..e145f7c6244 100644
--- a/assets/src/stories-editor/components/index.js
+++ b/assets/src/stories-editor/components/index.js
@@ -30,6 +30,7 @@ export { default as withCroppedFeaturedImage } from './with-cropped-featured-ima
 export { default as withHasSelectedInnerBlock } from './higher-order/with-has-selected-inner-block';
 export { default as withPageNumber } from './higher-order/with-page-number';
 export { default as withRightClickHandler } from './higher-order/with-right-click-handler';
+export { default as withKeyboardNavigation } from './higher-order/with-keyboard-navigation-handler';
 export { default as withStoryFeaturedImageNotice } from './higher-order/with-story-featured-image-notice';
 export { default as withEditFeaturedImage } from './with-edit-featured-image';
 export { default as withCustomVideoBlockEdit } from './with-custom-video-block-edit';
diff --git a/assets/src/stories-editor/index.js b/assets/src/stories-editor/index.js
index 69f98ba9f29..3f9cc9da32a 100644
--- a/assets/src/stories-editor/index.js
+++ b/assets/src/stories-editor/index.js
@@ -39,6 +39,7 @@ import {
 	withCallToActionValidation,
 	withCroppedFeaturedImage,
 	withRightClickHandler,
+	withKeyboardNavigation,
 } from './components';
 import {
 	maybeEnqueueFontStyle,
@@ -317,6 +318,7 @@ addFilter( 'blocks.registerBlockType', 'ampStoryEditorBlocks/deprecateCoreBlocks
 addFilter( 'editor.BlockEdit', 'ampStoryEditorBlocks/addStorySettings', withAmpStorySettings );
 addFilter( 'editor.BlockEdit', 'ampStoryEditorBlocks/addPageNumber', withPageNumber );
 addFilter( 'editor.BlockEdit', 'ampStoryEditorBlocks/rightClickHandler', withRightClickHandler );
+addFilter( 'editor.BlockEdit', 'ampStoryEditorBlocks/keyboardHandler', withKeyboardNavigation );
 addFilter( 'editor.BlockEdit', 'ampStoryEditorBlocks/addEditFeaturedImage', withEditFeaturedImage );
 addFilter( 'editor.BlockEdit', 'ampEditorBlocks/addVideoBlockPreview', withCustomVideoBlockEdit, 9 );
 addFilter( 'editor.PostFeaturedImage', 'ampStoryEditorBlocks/addFeaturedImageNotice', withStoryFeaturedImageNotice );