SQLite: Coerce jsonb columns to json before returning to Go code (#3968)

* SQLite: Coerce jsonb columns to json before returning to Go code

This one follows up the discussion in #3953 to try and make the `jsonb`
data type in SQLite usable (see discussion there, but I believe that
it's currently not).

According the SQLite docs on JSONB [1], it's considered a format that's
internal to the database itself, and no attempt should be made to parse
it elsewhere:

> JSONB is not intended as an external format to be used by
> applications. JSONB is designed for internal use by SQLite only.
> Programmers do not need to understand the JSONB format in order to use
> it effectively. Applications should access JSONB only through the JSON
> SQL functions, not by looking at individual bytes of the BLOB.

Currently, when trying to use a `jsonb` column in SQLite, sqlc ends up
returning the internal binary data, which ends up being unparsable in Go:

    riverdrivertest.go:3030:
                Error Trace:    /Users/brandur/Documents/projects/river/internal/riverinternaltest/riverdrivertest/riverdrivertest.go:3030
                Error:          Not equal:
                                expected: []byte{0x7b, 0x22, 0x66, 0x6f, 0x6f, 0x22, 0x3a, 0x20, 0x22, 0x62, 0x61, 0x72, 0x22, 0x7d}
                                actual  : []byte{0x8c, 0x37, 0x66, 0x6f, 0x6f, 0x37, 0x62, 0x61, 0x72}

                                Diff:
                                --- Expected
                                +++ Actual
                                @@ -1,3 +1,3 @@
                                -([]uint8) (len=14) {
                                - 00000000  7b 22 66 6f 6f 22 3a 20  22 62 61 72 22 7d        |{"foo": "bar"}|
                                +([]uint8) (len=9) {
                                + 00000000  8c 37 66 6f 6f 37 62 61  72                       |.7foo7bar|
                                 }
                Test:           TestDriverRiverSQLite/QueueCreateOrSetUpdatedAt/InsertsANewQueueWithDefaultUpdatedAt

The fix is that we should make sure to coerce `jsonb` columns back to
`json` before returning. That's what this pull request does, intercepting
`SELECT *` and wrapping `jsonb` columns with a `json(...)` invocation.

I also assign `json` and `jsonb` a `[]byte` data type by default so they
don't end up as `any`, which isn't very useful. `[]byte` is consistent
with the default for `pgx/v5`.

[1] https://sqlite.org/jsonb.html

* Make SQLite json/jsonb values `json.RawMessage` instead of `[]byte`

This in response to code review feedback, it's a little more correct to
make json/jsonb values `json.RawMessage` instead of `[]byte`. The former
is a form of the latter, but better represents that the value is meant
to be JSON.

* Introduce `Selector` interface for generating column expressions

This is response to code review feedback, add a new `Selector `interface
whose job it is to provide an engine-agnostic way of generating output
expressions for when selecting column values with `SELECT ...` or
`RETURNING ...`. This is exclusively needed for SQLite for the time
being, which uses it to wrap all output `jsonb` column values with a
call to `json(...)` so that values are coerced to a publicly usable
format before being returned.

[1] #3968 (comment)

* Make selectors internal to the `compiler` package

This is based on original code review feedback that I'd misread
initially. The selector interface doesn't need to be an outside package
for any reason (it's used only internal to the compiler), and this lets
us improve it somewhat by taking a full `*Column` struct rather than
having to make it a `dataType string` (because `Column` is internal to
`compiler` and it would otherwise introduce dependency cycles).

Author: brandur

internal/codegen/golang/sqlite_type.go

@@ -56,6 +56,9 @@ func sqliteType(req *plugin.GenerateRequest, options *opts.Options, col *plugin.
 		}
 		return "sql.NullTime"
 
+	case "json", "jsonb":
+		return "json.RawMessage"
+
 	case "any":
 		return "interface{}"
 

internal/compiler/engine.go

@@ -23,6 +23,7 @@ type Compiler struct {
 	result   *Result
 	analyzer analyzer.Analyzer
 	client   dbmanager.Client
+	selector selector
 
 	schema []string
 }
@@ -39,12 +40,15 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err
 	case config.EngineSQLite:
 		c.parser = sqlite.NewParser()
 		c.catalog = sqlite.NewCatalog()
+		c.selector = newSQLiteSelector()
 	case config.EngineMySQL:
 		c.parser = dolphin.NewParser()
 		c.catalog = dolphin.NewCatalog()
+		c.selector = newDefaultSelector()
 	case config.EnginePostgreSQL:
 		c.parser = postgresql.NewParser()
 		c.catalog = postgresql.NewCatalog()
+		c.selector = newDefaultSelector()
 		if conf.Database != nil {
 			if conf.Analyzer.Database == nil || *conf.Analyzer.Database {
 				c.analyzer = analyzer.Cached(

internal/compiler/expand.go

@@ -149,6 +149,11 @@ func (c *Compiler) expandStmt(qc *QueryCatalog, raw *ast.RawStmt, node ast.Node)
 				if counts[cname] > 1 {
 					cname = tableName + "." + cname
 				}
+
+				// This is important for SQLite in particular which needs to
+				// wrap jsonb column values with `json(colname)` so they're in a
+				// publicly usable format (i.e. not jsonb).
+				cname = c.selector.ColumnExpr(cname, column)
 				cols = append(cols, cname)
 			}
 		}

internal/compiler/selector.go

@@ -0,0 +1,46 @@
+package compiler
+
+// selector is an interface used by a compiler for generating expressions for
+// output columns in a `SELECT ...` or `RETURNING ...` statement.
+//
+// This interface is exclusively needed at the moment for SQLite, which must
+// wrap output `jsonb` columns with a `json(column_name)` invocation so that a
+// publicly consumable format (i.e. not jsonb) is returned.
+type selector interface {
+	// ColumnExpr generates output to be used in a `SELECT ...` or `RETURNING
+	// ...` statement based on input column name and metadata.
+	ColumnExpr(name string, column *Column) string
+}
+
+// defaultSelector is a selector implementation that does the simpliest possible
+// pass through when generating column expressions. Its use is suitable for all
+// database engines not requiring additional customization.
+type defaultSelector struct{}
+
+func newDefaultSelector() *defaultSelector {
+	return &defaultSelector{}
+}
+
+func (s *defaultSelector) ColumnExpr(name string, column *Column) string {
+	return name
+}
+
+type sqliteSelector struct{}
+
+func newSQLiteSelector() *sqliteSelector {
+	return &sqliteSelector{}
+}
+
+func (s *sqliteSelector) ColumnExpr(name string, column *Column) string {
+	// Under SQLite, neither json nor jsonb are real data types, and rather just
+	// of type blob, so database drivers just return whatever raw binary is
+	// stored as values. This is a problem for jsonb, which is considered an
+	// internal format to SQLite and no attempt should be made to parse it
+	// outside of the database itself. For jsonb columns in SQLite, wrap values
+	// in `json(col)` to coerce the internal binary format to JSON parsable by
+	// the user-space application.
+	if column.DataType == "jsonb" {
+		return "json(" + name + ")"
+	}
+	return name
+}

internal/compiler/selector_test.go

@@ -0,0 +1,35 @@
+package compiler
+
+import "testing"
+
+func TestSelector(t *testing.T) {
+	t.Parallel()
+
+	selectorExpectColumnExpr := func(t *testing.T, selector selector, expected, name string, column *Column) {
+		if actual := selector.ColumnExpr(name, column); expected != actual {
+			t.Errorf("Expected %v, got %v for data type %v", expected, actual, column.DataType)
+		}
+	}
+
+	t.Run("DefaultSelectorColumnExpr", func(t *testing.T) {
+		t.Parallel()
+
+		selector := newDefaultSelector()
+
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "integer"})
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "json"})
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "jsonb"})
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "text"})
+	})
+
+	t.Run("SQLiteSelectorColumnExpr", func(t *testing.T) {
+		t.Parallel()
+
+		selector := newSQLiteSelector()
+
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "integer"})
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "json"})
+		selectorExpectColumnExpr(t, selector, "json(my_column)", "my_column", &Column{DataType: "jsonb"})
+		selectorExpectColumnExpr(t, selector, "my_column", "my_column", &Column{DataType: "text"})
+	})
+}

internal/endtoend/testdata/jsonb/pgx/go/db.go

@@ -0,0 +1,15 @@
+-- name: InsertFoo :exec
+INSERT INTO foo (
+    a,
+    b,
+    c,
+    d
+) VALUES (
+    @a,
+    @b,
+    @c,
+    @d
+) RETURNING *;
+
+-- name: SelectFoo :exec
+SELECT * FROM foo;

internal/endtoend/testdata/jsonb/pgx/go/models.go

@@ -0,0 +1,7 @@
+CREATE TABLE foo (
+    a json not null,
+    b jsonb not null,
+    c json,
+    d jsonb
+);
+