Support both fenced and regular code blocks (#676)

Work in progress.

pkg/document/block.go

@@ -3,6 +3,7 @@ package document
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"math"
 	"regexp"
 	"strconv"
@@ -42,11 +43,21 @@ func (b CodeBlocks) Names() (result []string) {
 
 type renderer func(ast.Node, []byte) ([]byte, error)
 
+type CodeBlockEncoding int
+
+const (
+	Fenced CodeBlockEncoding = iota + 1
+	UnfencedWithSpaces
+	// todo(sebastian): goldmark converts all tabs to spaces
+	// UnfencedWithTab
+)
+
 type CodeBlock struct {
-	id            string
-	idGenerated   bool
 	attributes    map[string]string
 	document      *Document
+	encoding      CodeBlockEncoding
+	id            string
+	idGenerated   bool
 	inner         *ast.FencedCodeBlock
 	intro         string // paragraph immediately before the code block
 	language      string
@@ -60,41 +71,59 @@ var _ Block = (*CodeBlock)(nil)
 
 func newCodeBlock(
 	document *Document,
-	node *ast.FencedCodeBlock,
+	node ast.Node,
 	identityResolver identityResolver,
 	nameResolver *nameResolver,
 	source []byte,
 	render renderer,
 ) (*CodeBlock, error) {
-	attributes, err := getAttributes(node, source, DefaultAttributeParser)
+	var fenced *ast.FencedCodeBlock
+	encoding := Fenced
+
+	switch node.Kind() {
+	case ast.KindCodeBlock:
+		// todo(sebastian): should we attempt to preserve tab vs spaces?
+		encoding = UnfencedWithSpaces
+		fenced = ast.NewFencedCodeBlock(ast.NewText())
+		fenced.BaseBlock = node.(*ast.CodeBlock).BaseBlock
+	case ast.KindFencedCodeBlock:
+		fenced = node.(*ast.FencedCodeBlock)
+	default:
+		return nil, errors.New("invalid node kind neither CodeBlock nor FencedCodeBlock")
+	}
+
+	attributes, err := getAttributes(fenced, source, DefaultAttributeParser)
 	if err != nil {
 		return nil, err
 	}
 
-	id, hasID := identityResolver.GetCellID(node, attributes)
+	id, hasID := identityResolver.GetCellID(fenced, attributes)
 
-	name, hasName := getName(node, source, nameResolver, attributes)
+	name, hasName := getName(fenced, source, nameResolver, attributes)
 
-	value, err := render(node, source)
+	value, err := render(fenced, source)
 	if err != nil {
 		return nil, err
 	}
 
 	return &CodeBlock{
-		id:            id,
-		idGenerated:   !hasID,
 		attributes:    attributes,
 		document:      document,
-		inner:         node,
-		intro:         getIntro(node, source),
-		language:      getLanguage(node, source),
-		lines:         getLines(node, source),
+		encoding:      encoding,
+		id:            id,
+		idGenerated:   !hasID,
+		inner:         fenced,
+		intro:         getIntro(fenced, source),
+		language:      getLanguage(fenced, source),
+		lines:         getLines(fenced, source),
 		name:          name,
 		nameGenerated: !hasName,
 		value:         value,
 	}, nil
 }
 
+func (b *CodeBlock) FencedEncoding() bool { return b.encoding == Fenced }
+
 func (b *CodeBlock) Attributes() map[string]string { return b.attributes }
 
 func (b *CodeBlock) Background() bool {

pkg/document/document.go

@@ -171,10 +171,10 @@ func (d *Document) parse() {
 func (d *Document) buildBlocksTree(parent ast.Node, node *Node) error {
 	for astNode := parent.FirstChild(); astNode != nil; astNode = astNode.NextSibling() {
 		switch astNode.Kind() {
-		case ast.KindFencedCodeBlock:
+		case ast.KindCodeBlock, ast.KindFencedCodeBlock:
 			block, err := newCodeBlock(
 				d,
-				astNode.(*ast.FencedCodeBlock),
+				astNode,
 				d.identityResolver,
 				d.nameResolver,
 				d.content,

pkg/document/document_test.go

@@ -47,7 +47,7 @@ First paragraph.
 	assert.Len(t, node.children[2].children[0].children, 0)
 	assert.Equal(t, "bq1\n", string(node.children[2].children[0].Item().Value()))
 	assert.Len(t, node.children[2].children[1].children, 0)
-	assert.Equal(t, "    echo \"inside bq\"\n", string(node.children[2].children[1].Item().Value()))
+	assert.Equal(t, "```\necho \"inside bq\"\n```\n", string(node.children[2].children[1].Item().Value()))
 	assert.Len(t, node.children[2].children[2].children, 0)
 	assert.Equal(t, "bq2\nbq3\n", string(node.children[2].children[2].Item().Value()))
 	assert.Len(t, node.children[3].children, 3)

pkg/document/editor/cell.go

@@ -198,6 +198,10 @@ func toCellsRec(
 			}
 			metadata[PrefixAttributeName(InternalAttributePrefix, "nameGenerated")] = nameGeneratedStr
 
+			if !block.FencedEncoding() {
+				metadata[PrefixAttributeName(InternalAttributePrefix, "fenced")] = "false"
+			}
+
 			*cells = append(*cells, &Cell{
 				Kind:       CodeKind,
 				Value:      string(block.Content()),
@@ -320,29 +324,9 @@ func serializeCells(cells []*Cell) []byte {
 	var buf bytes.Buffer
 
 	for idx, cell := range cells {
-		value := cell.Value
-
 		switch cell.Kind {
 		case CodeKind:
-			ticksCount := longestBacktickSeq(value)
-			if ticksCount < 3 {
-				ticksCount = 3
-			}
-
-			_, _ = buf.Write(bytes.Repeat([]byte{'`'}, ticksCount))
-			_, _ = buf.WriteString(cell.LanguageID)
-
-			serializeFencedCodeAttributes(&buf, cell)
-
-			_ = buf.WriteByte('\n')
-			_, _ = buf.WriteString(cell.Value)
-			_ = buf.WriteByte('\n')
-
-			serializeCellOutputsText(&buf, cell)
-
-			_, _ = buf.Write(bytes.Repeat([]byte{'`'}, ticksCount))
-
-			serializeCellOutputsImage(&buf, cell)
+			serializeCellCodeBlock(&buf, cell)
 
 		case MarkupKind:
 			_, _ = buf.WriteString(cell.Value)
@@ -361,6 +345,43 @@ func serializeCells(cells []*Cell) []byte {
 	return buf.Bytes()
 }
 
+func serializeCellCodeBlock(w io.Writer, cell *Cell) {
+	var buf bytes.Buffer
+	value := cell.Value
+
+	if b, err := strconv.ParseBool(cell.Metadata[PrefixAttributeName(InternalAttributePrefix, "fenced")]); err == nil && !b {
+		for _, v := range strings.Split(value, "\n") {
+			_, _ = buf.Write(bytes.Repeat([]byte{' '}, 4))
+			_, _ = buf.WriteString(v)
+			_ = buf.WriteByte('\n')
+		}
+
+		serializeCellOutputsText(&buf, cell)
+	} else {
+		ticksCount := longestBacktickSeq(value)
+		if ticksCount < 3 {
+			ticksCount = 3
+		}
+
+		_, _ = buf.Write(bytes.Repeat([]byte{'`'}, ticksCount))
+		_, _ = buf.WriteString(cell.LanguageID)
+
+		serializeFencedCodeAttributes(&buf, cell)
+
+		_ = buf.WriteByte('\n')
+		_, _ = buf.WriteString(cell.Value)
+		_ = buf.WriteByte('\n')
+
+		serializeCellOutputsText(&buf, cell)
+
+		_, _ = buf.Write(bytes.Repeat([]byte{'`'}, ticksCount))
+	}
+
+	serializeCellOutputsImage(&buf, cell)
+
+	_, _ = w.Write(buf.Bytes())
+}
+
 func serializeCellOutputsText(w io.Writer, cell *Cell) {
 	var buf bytes.Buffer
 	for _, output := range cell.Outputs {

pkg/document/editor/cell_test.go

@@ -49,12 +49,12 @@ It can have an annotation with a name:
 $ echo "Hello, runme!"
 ` + "```" + `
 
-> bq 1
-> bq 2
->
->     echo 1
->
-> b1 3
+bq 1
+bq 2
+
+    echo 1
+
+b1 3
 
 1. Item 1
 
@@ -77,17 +77,19 @@ func Test_toCells_DataNested(t *testing.T) {
 	node, err := doc.Root()
 	require.NoError(t, err)
 	cells := toCells(doc, node, testDataNested)
-	assert.Len(t, cells, 10)
+	assert.Len(t, cells, 12)
 	assert.Equal(t, "# Examples", cells[0].Value)
 	assert.Equal(t, "It can have an annotation with a name:", cells[1].Value)
 	assert.Equal(t, "$ echo \"Hello, runme!\"", cells[2].Value)
-	assert.Equal(t, "> bq 1\n> bq 2\n>\n>     echo 1\n>\n> b1 3", cells[3].Value)
-	assert.Equal(t, "1. Item 1", cells[4].Value)
-	assert.Equal(t, "$ echo \"Hello, runme!\"", cells[5].Value)
-	assert.Equal(t, "First inner paragraph", cells[6].Value)
-	assert.Equal(t, "Second inner paragraph", cells[7].Value)
-	assert.Equal(t, "2. Item 2", cells[8].Value)
-	assert.Equal(t, "3. Item 3", cells[9].Value)
+	assert.Equal(t, "bq 1\nbq 2", cells[3].Value)
+	assert.Equal(t, "echo 1", cells[4].Value)
+	assert.Equal(t, "b1 3", cells[5].Value)
+	assert.Equal(t, "1. Item 1", cells[6].Value)
+	assert.Equal(t, "$ echo \"Hello, runme!\"", cells[7].Value)
+	assert.Equal(t, "First inner paragraph", cells[8].Value)
+	assert.Equal(t, "Second inner paragraph", cells[9].Value)
+	assert.Equal(t, "2. Item 2", cells[10].Value)
+	assert.Equal(t, "3. Item 3", cells[11].Value)
 }
 
 func Test_toCells_Lists(t *testing.T) {

pkg/document/editor/editor_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -675,3 +676,61 @@ func TestEditor_CodeBlockTransformation(t *testing.T) {
 		)
 	})
 }
+
+func TestEditor_CodeBlockEndcoding(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Fenced", func(t *testing.T) {
+		original := []byte(`## Tests
+
+Run all tests:
+
+` + strings.Repeat("`", 3) + `sh
+export TESTING="1"
+dagger call test all
+` + strings.Repeat("`", 3) + `
+
+Run a specific test:
+
+` + strings.Repeat("`", 3) + `sh
+dagger call test specific --pkg="./core/integration" --run="^TestModule/TestNamespacing$"
+` + strings.Repeat("`", 3) + `
+
+A paragraph
+`)
+		notebook, err := Deserialize(original, Options{IdentityResolver: identityResolverNone})
+		require.NoError(t, err)
+		result, err := Serialize(notebook, nil, Options{})
+		require.NoError(t, err)
+		assert.Equal(
+			t,
+			string(original),
+			string(result),
+		)
+	})
+
+	t.Run("UnfencedWithSpaces", func(t *testing.T) {
+		original := []byte(`## Tests
+
+Run all tests:
+
+    export TESTING="1"
+    dagger call test all
+
+Run a specific test:
+
+    dagger call test specific --pkg="./core/integration" --run="^TestModule/TestNamespacing$"
+
+A paragraph
+`)
+		notebook, err := Deserialize(original, Options{IdentityResolver: identityResolverNone})
+		require.NoError(t, err)
+		result, err := Serialize(notebook, nil, Options{})
+		require.NoError(t, err)
+		assert.Equal(
+			t,
+			string(original),
+			string(result),
+		)
+	})
+}

pkg/document/node_test.go

@@ -46,9 +46,9 @@ func TestCollectCodeBlocks(t *testing.T) {
 	node, err := doc.Root()
 	require.NoError(t, err)
 	codeBlocks := CollectCodeBlocks(node)
-	assert.Len(t, codeBlocks, 2)
-	assert.Equal(t, "```sh {name=echo first= second=2}\n$ echo \"Hello, runme!\"\n```\n", string(codeBlocks[0].Value()))
-	assert.Equal(t, "```sh\necho 1\n```\n", string(codeBlocks[1].Value()))
+	assert.Len(t, codeBlocks, 3)
+	assert.Equal(t, "```sh {name=echo first= second=2}\n$ echo \"Hello, runme!\"\n```\n", string(codeBlocks[1].Value()))
+	assert.Equal(t, "```sh\necho 1\n```\n", string(codeBlocks[2].Value()))
 }
 
 func TestCodeBlock_Intro(t *testing.T) {