diff --git a/pkg/config/spec.go b/pkg/config/spec.go
index 842b6244..c50b31b6 100644
--- a/pkg/config/spec.go
+++ b/pkg/config/spec.go
@@ -403,6 +403,14 @@ func NewUpgradeSpec(cfg *Config) (*v1.UpgradeSpec, error) {
 func setUpgradeSourceSize(cfg *Config, spec *v1.UpgradeSpec) error {
 	var size int64
 	var err error
+	var originalSize int64
+
+	// Store the default given size in the spec. This includes the user specified values which have already been marshalled in the spec
+	if spec.RecoveryUpgrade() {
+		originalSize = int64(spec.Recovery.Size)
+	} else {
+		originalSize = int64(spec.Active.Size)
+	}
 
 	var targetSpec *v1.Image
 	if spec.RecoveryUpgrade() {
@@ -420,8 +428,15 @@ func setUpgradeSourceSize(cfg *Config, spec *v1.UpgradeSpec) error {
 		return err
 	}
 
-	cfg.Logger.Infof("Setting image size to %dMb", size)
-	targetSpec.Size = uint(size)
+	if size < originalSize {
+		cfg.Logger.Debugf("Calculated size (%dMb) is less than specified/default size (%dMb)", size, originalSize)
+		targetSpec.Size = uint(originalSize)
+	} else {
+		cfg.Logger.Debugf("Calculated size (%dMb) is higher than specified/default size (%dMb)", size, originalSize)
+		targetSpec.Size = uint(size)
+	}
+
+	cfg.Logger.Infof("Setting image size to %dMb", targetSpec.Size)
 
 	return nil
 }