Use wazero-based libpgquery wrapper in currently disabled environments (

#3027) * Use wazero-based wrapper of libpgquery when cgo is disabled or on windows
sqlc-dev · Dec 5, 2023 · 4a05999 · 4a05999
1 parent 7370598
commit 4a05999
Show file tree

Hide file tree

Showing 19 changed files with 115 additions and 137 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,31 +5,24 @@ on:
       - main
   pull_request:
 jobs:
-  windows-build:
-    if: ${{ github.ref == 'refs/heads/main' }}
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-go@v4
-      with:
-        go-version-file: go.mod
-        check-latest: true
-    - run: go build ./...
-
-  darwin-build:
-    if: ${{ github.ref == 'refs/heads/main' }}
-    runs-on: macos-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-go@v4
-      with:
-        go-version-file: go.mod
-        check-latest: true
-    - run: go build ./...
-
-  build:
-    name: test
-    runs-on: ubuntu-latest
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        cgo: ['1', '0']
+        # Workaround no native support for conditional matrix items
+        # https://github.com/orgs/community/discussions/26253#discussioncomment-6745038
+        isMain: 
+          - ${{ github.ref == 'refs/heads/main' }}
+        exclude:
+          - isMain: false
+        include:
+          - os: ubuntu-latest
+            cgo: '1'
+          - os: ubuntu-latest
+            cgo: '0'
+    name: test ${{ matrix.os }} cgo=${{ matrix.cgo }}
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v4
@@ -46,17 +39,22 @@ jobs:
 
     - name: install ./...
       run: go install ./...
+      env:
+        CGO_ENABLED: ${{ matrix.cgo }}
 
     - name: build internal/endtoend
       run: go build ./...
       working-directory: internal/endtoend/testdata
+      env:
+        CGO_ENABLED: ${{ matrix.cgo }}
 
     - name: test ./...
       run: gotestsum --junitfile junit.xml -- --tags=examples -timeout 20m ./...
       env:
         CI_SQLC_PROJECT_ID: ${{ secrets.CI_SQLC_PROJECT_ID }}
         CI_SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
         SQLC_AUTH_TOKEN: ${{ secrets.CI_SQLC_AUTH_TOKEN }}
+        CGO_ENABLED: ${{ matrix.cgo }}
 
   vuln_check:
     runs-on: ubuntu-latest

diff --git a/examples/authors/sqlite/db_test.go b/examples/authors/sqlite/db_test.go
@@ -1,5 +1,5 @@
-//go:build examples
-// +build examples
+//go:build examples && cgo
+// +build examples,cgo
 
 package authors
 

diff --git a/examples/booktest/sqlite/db_test.go b/examples/booktest/sqlite/db_test.go
@@ -1,5 +1,5 @@
-//go:build examples
-// +build examples
+//go:build examples && cgo
+// +build examples,cgo
 
 package booktest
 

diff --git a/examples/ondeck/sqlite/db_test.go b/examples/ondeck/sqlite/db_test.go
@@ -1,5 +1,5 @@
-//go:build examples
-// +build examples
+//go:build examples && cgo
+// +build examples,cgo
 
 package ondeck
 

diff --git a/go.mod b/go.mod
@@ -15,18 +15,21 @@ require (
 	github.com/jinzhu/inflection v1.0.0
 	github.com/lib/pq v1.10.9
 	github.com/mattn/go-sqlite3 v1.14.18
-	github.com/pganalyze/pg_query_go/v4 v4.2.3
+	github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231205012101-7463430c7b73
 	github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1
 	github.com/riza-io/grpc-go v0.2.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
+	github.com/wasilibs/go-pgquery v0.0.0-20231205013331-96e794bb074e
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/sync v0.5.0
 	google.golang.org/grpc v1.59.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6 // indirect
+
 require (
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect

diff --git a/go.sum b/go.sum
@@ -29,21 +29,11 @@ github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/cel-go v0.18.2 h1:L0B6sNBSVmt0OyECi8v6VOS74KOc9W/tLiWKfZABvf4=
 github.com/google/cel-go v0.18.2/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -129,8 +119,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
 github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/pganalyze/pg_query_go/v4 v4.2.3 h1:cNLqyiVMasV7YGWyYV+fkXyHp32gDfXVNCqoHztEGNk=
-github.com/pganalyze/pg_query_go/v4 v4.2.3/go.mod h1:aEkDNOXNM5j0YGzaAapwJ7LB3dLNj+bvbWcLv1hOVqA=
+github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231205012101-7463430c7b73 h1:vZEujakYrBzd7CWMwqTsDRU7bdRm4Aiu76X9K3P+qtI=
+github.com/pganalyze/pg_query_go/v4 v4.2.4-0.20231205012101-7463430c7b73/go.mod h1:pRJ9rCcWIhDLAzymhH2iY5kEVEfU6UGrL1wBWuO7gmM=
 github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 h1:+FZIDR/D97YOPik4N4lPDaUcLDF/EQPogxtlHB2ZZRM=
@@ -185,6 +175,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/wasilibs/go-pgquery v0.0.0-20231205013331-96e794bb074e h1:sGIC6/D0KqpA+qBSDSVDQswU/IJVYkbnUXnipgTLQWk=
+github.com/wasilibs/go-pgquery v0.0.0-20231205013331-96e794bb074e/go.mod h1:KW0azBSWqkPZ71r+3O4qt8h6A/NisFLp0rbjZ3py4OE=
+github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6 h1:jwbU8u5TuXModzdEG4wI0g4FyuD7ROSttU86go5sPdU=
+github.com/wasilibs/wazerox v0.0.0-20231117065139-b3503f4aeff6/go.mod h1:IQNVyA4d1hWIe23mlMMuqXjyWMdndgSlNx6FqBkwPsM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
@@ -303,12 +297,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
 google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
 google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=

diff --git a/internal/codegen/sdk/utils.go b/internal/codegen/sdk/utils.go
@@ -23,12 +23,14 @@ func Title(s string) string {
 // a backtick, replace it the following way:
 //
 // input:
-// 	SELECT `group` FROM foo
+//
+//	SELECT `group` FROM foo
 //
 // output:
-// 	SELECT ` + "`" + `group` + "`" + ` FROM foo
 //
-// The escaped string must be rendered inside an existing string literal
+//	SELECT ` + "`" + `group` + "`" + ` FROM foo
+//
+// # The escaped string must be rendered inside an existing string literal
 //
 // A string cannot be escaped twice
 func EscapeBacktick(s string) string {

diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go
@@ -1,3 +1,7 @@
+// Currently requires cgo for wasmtime and has line-ending issues on windows.
+//go:build cgo && !windows
+// +build cgo,!windows
+
 package main
 
 import (

diff --git a/internal/endtoend/fmt_test.go b/internal/endtoend/fmt_test.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"testing"
 
-	pg_query "github.com/pganalyze/pg_query_go/v4"
 	"github.com/sqlc-dev/sqlc/internal/debug"
 	"github.com/sqlc-dev/sqlc/internal/engine/postgresql"
 	"github.com/sqlc-dev/sqlc/internal/sql/ast"
@@ -40,7 +39,7 @@ func TestFormat(t *testing.T) {
 				}
 				query := query
 				t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-					expected, err := pg_query.Fingerprint(string(query))
+					expected, err := postgresql.Fingerprint(string(query))
 					if err != nil {
 						t.Fatal(err)
 					}
@@ -52,12 +51,12 @@ func TestFormat(t *testing.T) {
 						t.Fatal("expected one statement")
 					}
 					if false {
-						r, err := pg_query.Parse(string(query))
+						r, err := postgresql.Parse(string(query))
 						debug.Dump(r, err)
 					}
 
 					out := ast.Format(stmts[0].Raw)
-					actual, err := pg_query.Fingerprint(out)
+					actual, err := postgresql.Fingerprint(out)
 					if err != nil {
 						t.Error(err)
 					}

diff --git a/internal/engine/postgresql/convert.go b/internal/engine/postgresql/convert.go
@@ -1,6 +1,3 @@
-//go:build !windows && cgo
-// +build !windows,cgo
-
 package postgresql
 
 import (

diff --git a/internal/engine/postgresql/parse.go b/internal/engine/postgresql/parse.go
@@ -1,6 +1,3 @@
-//go:build !windows && cgo
-// +build !windows,cgo
-
 package postgresql
 
 import (
@@ -10,8 +7,8 @@ import (
 	"strings"
 
 	nodes "github.com/pganalyze/pg_query_go/v4"
-	"github.com/pganalyze/pg_query_go/v4/parser"
 
+	"github.com/sqlc-dev/sqlc/internal/engine/postgresql/parser"
 	"github.com/sqlc-dev/sqlc/internal/source"
 	"github.com/sqlc-dev/sqlc/internal/sql/ast"
 	"github.com/sqlc-dev/sqlc/internal/sql/sqlerr"
@@ -154,7 +151,7 @@ func (p *Parser) Parse(r io.Reader) ([]ast.Statement, error) {
 	if err != nil {
 		return nil, err
 	}
-	tree, err := nodes.Parse(string(contents))
+	tree, err := Parse(string(contents))
 	if err != nil {
 		pErr := normalizeErr(err)
 		return nil, pErr

diff --git a/internal/engine/postgresql/parse_default.go b/internal/engine/postgresql/parse_default.go
@@ -0,0 +1,11 @@
+//go:build !windows && cgo
+// +build !windows,cgo
+
+package postgresql
+
+import (
+	nodes "github.com/pganalyze/pg_query_go/v4"
+)
+
+var Parse = nodes.Parse
+var Fingerprint = nodes.Fingerprint
diff --git a/internal/engine/postgresql/parse_disabled.go b/internal/engine/postgresql/parse_disabled.go
diff --git a/internal/engine/postgresql/parse_wasi.go b/internal/engine/postgresql/parse_wasi.go
@@ -0,0 +1,11 @@
+//go:build windows || !cgo
+// +build windows !cgo
+
+package postgresql
+
+import (
+	nodes "github.com/wasilibs/go-pgquery"
+)
+
+var Parse = nodes.Parse
+var Fingerprint = nodes.Fingerprint
diff --git a/internal/engine/postgresql/parser/parser_default.go b/internal/engine/postgresql/parser/parser_default.go
@@ -0,0 +1,8 @@
+//go:build !windows && cgo
+// +build !windows,cgo
+
+package parser
+
+import "github.com/pganalyze/pg_query_go/v4/parser"
+
+type Error = parser.Error
diff --git a/internal/engine/postgresql/parser/parser_wasi.go b/internal/engine/postgresql/parser/parser_wasi.go
@@ -0,0 +1,8 @@
+//go:build windows || !cgo
+// +build windows !cgo
+
+package parser
+
+import "github.com/wasilibs/go-pgquery/parser"
+
+type Error = parser.Error
diff --git a/internal/engine/postgresql/utils.go b/internal/engine/postgresql/utils.go
@@ -1,6 +1,3 @@
-//go:build !windows && cgo
-// +build !windows,cgo
-
 package postgresql
 
 import (

diff --git a/internal/sql/sqlpath/read.go b/internal/sql/sqlpath/read.go
@@ -46,7 +46,7 @@ func Glob(patterns []string) ([]string, error) {
 				files = append(files, filepath.Join(path, f.Name()))
 			}
 		} else {
-			files = append(files, path)
+			files = append(files, filepath.Clean(path))
 		}
 	}
 	var sqlFiles []string