From 17e4e7783e4daa8d883ec62ca98e6db6b0a3b9d0 Mon Sep 17 00:00:00 2001 From: thxCode Date: Mon, 12 Jun 2023 16:42:21 +0800 Subject: [PATCH] fix: import with lock sql file Signed-off-by: thxCode --- byteset/provider_test.go | 11 --- byteset/resource_pipeline.go | 56 ++++++------ byteset/resource_pipeline_test.go | 68 ++++++++++----- byteset/testdata/mysql-fk.sql | 4 - byteset/testdata/mysql.sql | 4 +- byteset/testdata/postgres-fk.sql | 4 - byteset/testdata/postgres.sql | 4 +- byteset/testdata/sqlite-fk.sql | 4 - byteset/testdata/sqlite.sql | 2 - docs/index.md | 4 +- docs/resources/pipeline.md | 7 +- examples/sqlite/main.tf | 4 +- go.mod | 7 +- go.sum | 21 +++-- main.go | 4 +- pipeline/destination.go | 138 ++++++++++++++++++++++++++---- pipeline/option.go | 6 +- pipeline/source.go | 103 ++++++---------------- pipeline/source_file.go | 86 +++++++++++++++++++ pipeline/source_file_test.go | 80 +++++++++++++++++ pipeline/testdata/complex.sql | 47 ++++++++++ utils/signalx/doc.go | 7 ++ utils/signalx/signal.go | 33 +++++++ utils/signalx/signal_posix.go | 13 +++ utils/signalx/signal_windows.go | 10 +++ utils/sqlx/driver.go | 6 ++ utils/sqlx/helper.go | 76 ++++++++++++++++ utils/testx/file.go | 19 ++++ 28 files changed, 643 insertions(+), 185 deletions(-) create mode 100644 pipeline/source_file.go create mode 100644 pipeline/source_file_test.go create mode 100644 pipeline/testdata/complex.sql create mode 100644 utils/signalx/doc.go create mode 100644 utils/signalx/signal.go create mode 100644 utils/signalx/signal_posix.go create mode 100644 utils/signalx/signal_windows.go create mode 100644 utils/sqlx/helper.go create mode 100644 utils/testx/file.go diff --git a/byteset/provider_test.go b/byteset/provider_test.go index acc9588..f6fc49a 100644 --- a/byteset/provider_test.go +++ b/byteset/provider_test.go @@ -4,8 +4,6 @@ import ( "context" "html/template" "io" - "os" - "path/filepath" "strings" "testing" @@ -60,15 +58,6 @@ func renderConfigTemplate(ct string, keyValuePairs ...any) string { return s.String() } -func testdataPath() string { - dir, err := os.Getwd() - if err != nil { - panic(err) - } - - return filepath.Join(dir, "testdata") -} - type dockerContainer struct { Name string Image string diff --git a/byteset/resource_pipeline.go b/byteset/resource_pipeline.go index 150f45e..6948bc4 100644 --- a/byteset/resource_pipeline.go +++ b/byteset/resource_pipeline.go @@ -5,10 +5,12 @@ import ( "strings" "time" + "github.com/hashicorp/terraform-plugin-framework-validators/int64validator" "github.com/hashicorp/terraform-plugin-framework/path" "github.com/hashicorp/terraform-plugin-framework/resource" "github.com/hashicorp/terraform-plugin-framework/resource/schema" "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64default" + "github.com/hashicorp/terraform-plugin-framework/schema/validator" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-log/tflog" @@ -41,7 +43,6 @@ type ResourcePipelineDestination struct { Address types.String `tfsdk:"address"` ConnMaxOpen types.Int64 `tfsdk:"conn_max_open"` ConnMaxIdle types.Int64 `tfsdk:"conn_max_idle"` - ConnMaxLife types.Int64 `tfsdk:"conn_max_life"` Salt types.String `tfsdk:"salt"` } @@ -51,9 +52,6 @@ func (r ResourcePipelineDestination) Reflect(ctx context.Context) (pipeline.Dest r.Address.ValueString(), pipeline.WithConnMaxOpen(int(r.ConnMaxOpen.ValueInt64())), pipeline.WithConnMaxIdle(int(r.ConnMaxIdle.ValueInt64())), - pipeline.WithConnMaxLife( - time.Duration(r.ConnMaxLife.ValueInt64())*time.Second, - ), ) } @@ -118,21 +116,27 @@ choose from local/remote SQL file or database. "conn_max_open": schema.Int64Attribute{ Optional: true, Computed: true, - Default: int64default.StaticInt64(15), + Default: int64default.StaticInt64(5), Description: `The maximum opening connectors of source database.`, + Validators: []validator.Int64{ + int64validator.AtLeast(1), + }, }, "conn_max_idle": schema.Int64Attribute{ Optional: true, Computed: true, Default: int64default.StaticInt64(5), Description: `The maximum idling connections of source database.`, + Validators: []validator.Int64{ + int64validator.AtLeast(1), + int64validator.AtMostSumOf( + path.MatchRelative().AtParent().AtName("conn_max_open")), + }, }, "conn_max_life": schema.Int64Attribute{ - Optional: true, - Computed: true, - Default: int64default.StaticInt64( - 5 * 60, - ), + Optional: true, + Computed: true, + Default: int64default.StaticInt64(5 * 60), Description: `The maximum lifetime in seconds of source database.`, }, }, @@ -153,24 +157,28 @@ choose from local/remote SQL file or database. - mssql://[username:[password]@][address][:port][/instance][?database=dbname¶m1=value1&...]`, }, "conn_max_open": schema.Int64Attribute{ - Optional: true, - Computed: true, - Default: int64default.StaticInt64(15), - Description: `The maximum opening connectors of destination database.`, + Optional: true, + Computed: true, + Default: int64default.StaticInt64(5), + Description: `The maximum opening connectors of destination database, +if the given SQL file is using single transaction, should turn down the "conn_max_open" to 1. +`, + Validators: []validator.Int64{ + int64validator.AtLeast(1), + }, }, "conn_max_idle": schema.Int64Attribute{ - Optional: true, - Computed: true, - Default: int64default.StaticInt64(5), - Description: `The maximum idling connections of destination database.`, - }, - "conn_max_life": schema.Int64Attribute{ Optional: true, Computed: true, - Default: int64default.StaticInt64( - 5 * 60, - ), - Description: `The maximum lifetime in seconds of destination database.`, + Default: int64default.StaticInt64(5), + Description: `The maximum idling connections of destination database, +if the given SQL file is using single transaction, should turn down the "conn_max_idle" to 1. +`, + Validators: []validator.Int64{ + int64validator.AtLeast(1), + int64validator.AtMostSumOf( + path.MatchRelative().AtParent().AtName("conn_max_open")), + }, }, "salt": schema.StringAttribute{ Optional: true, diff --git a/byteset/resource_pipeline_test.go b/byteset/resource_pipeline_test.go index 52d1ced..594d346 100644 --- a/byteset/resource_pipeline_test.go +++ b/byteset/resource_pipeline_test.go @@ -8,17 +8,20 @@ import ( "github.com/hashicorp/terraform-plugin-testing/helper/resource" "github.com/seal-io/terraform-provider-byteset/utils/strx" + "github.com/seal-io/terraform-provider-byteset/utils/testx" ) -func TestAccResourcePipeline_sqlite(t *testing.T) { +func TestAccResourcePipeline_file_to_sqlite(t *testing.T) { // Test pipeline. var ( + testdataPath = testx.AbsolutePath("testdata") + resourceName = "byteset_pipeline.test" - basicSrc = fmt.Sprintf("file://%s/sqlite.sql", testdataPath()) + basicSrc = fmt.Sprintf("file://%s/sqlite.sql", testdataPath) basicDst = "sqlite:///tmp/sqlite.db" - fkSrc = fmt.Sprintf("file://%s/sqlite-fk.sql", testdataPath()) + fkSrc = fmt.Sprintf("file://%s/sqlite-fk.sql", testdataPath) fkDst = "sqlite:///tmp/sqlite.db?_pragma=foreign_keys(1)" ) @@ -28,28 +31,29 @@ func TestAccResourcePipeline_sqlite(t *testing.T) { Steps: []resource.TestStep{ // Basic. { - Config: testConfig(basicSrc, basicDst), + Config: testConfigOfSourceFile(basicSrc, basicDst, 1, 1), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", basicSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", basicDst), - resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "15"), - resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), - resource.TestCheckResourceAttr(resourceName, "destination.conn_max_life", "300"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "1"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "1"), ), }, // Foreign Key. { - Config: testConfig(fkSrc, fkDst), + Config: testConfigOfSourceFile(fkSrc, fkDst, 1, 1), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", fkSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", fkDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "1"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "1"), ), }, }, }) } -func TestAccResourcePipeline_mysql(t *testing.T) { +func TestAccResourcePipeline_file_to_mysql(t *testing.T) { // Start Database. var ( database = "byteset" @@ -78,12 +82,13 @@ func TestAccResourcePipeline_mysql(t *testing.T) { // Test pipeline. var ( + testdataPath = testx.AbsolutePath("testdata") resourceName = "byteset_pipeline.test" - basicSrc = fmt.Sprintf("file://%s/mysql.sql", testdataPath()) + basicSrc = fmt.Sprintf("file://%s/mysql.sql", testdataPath) basicDst = fmt.Sprintf("mysql://root:%s@tcp(127.0.0.1:3306)/%s", password, database) - fkSrc = fmt.Sprintf("file://%s/mysql-fk.sql", testdataPath()) + fkSrc = fmt.Sprintf("file://%s/mysql-fk.sql", testdataPath) fkDst = fmt.Sprintf("mysql://root:%s@tcp(127.0.0.1)/%s", password, database) largeSrc = "https://raw.githubusercontent.com/seal-io/terraform-provider-byteset/main/byteset/testdata/mysql-lg.sql" @@ -95,31 +100,37 @@ func TestAccResourcePipeline_mysql(t *testing.T) { ProtoV6ProviderFactories: testAccProviderFactories, Steps: []resource.TestStep{ { - Config: testConfig(basicSrc, basicDst), + Config: testConfigOfSourceFile(basicSrc, basicDst, 5, 5), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", basicSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", basicDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "5"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), ), }, { - Config: testConfig(fkSrc, fkDst), + Config: testConfigOfSourceFile(fkSrc, fkDst, 5, 5), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", fkSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", fkDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "5"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), ), }, { - Config: testConfig(largeSrc, largeDst), + Config: testConfigOfSourceFile(largeSrc, largeDst, 5, 5), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", largeSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", largeDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "5"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), ), }, }, }) } -func TestAccResourcePipeline_postgres(t *testing.T) { +func TestAccResourcePipeline_file_to_postgres(t *testing.T) { // Start Database. var ( database = "byteset" @@ -149,12 +160,13 @@ func TestAccResourcePipeline_postgres(t *testing.T) { // Test pipeline. var ( + testdataPath = testx.AbsolutePath("testdata") resourceName = "byteset_pipeline.test" - basicSrc = fmt.Sprintf("file://%s/postgres.sql", testdataPath()) + basicSrc = fmt.Sprintf("file://%s/postgres.sql", testdataPath) basicDst = fmt.Sprintf("postgres://root:%s@127.0.0.1:5432/%s?sslmode=disable", password, database) - fkSrc = fmt.Sprintf("file://%s/postgres-fk.sql", testdataPath()) + fkSrc = fmt.Sprintf("file://%s/postgres-fk.sql", testdataPath) fkDst = fmt.Sprintf("postgres://root:%s@127.0.0.1/%s?sslmode=disable", password, database) largeSrc = "https://raw.githubusercontent.com/seal-io/terraform-provider-byteset/main/byteset/testdata/postgres-lg.sql" @@ -166,31 +178,37 @@ func TestAccResourcePipeline_postgres(t *testing.T) { ProtoV6ProviderFactories: testAccProviderFactories, Steps: []resource.TestStep{ { - Config: testConfig(basicSrc, basicDst), + Config: testConfigOfSourceFile(basicSrc, basicDst, 5, 5), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", basicSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", basicDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "5"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), ), }, { - Config: testConfig(fkSrc, fkDst), + Config: testConfigOfSourceFile(fkSrc, fkDst, 5, 5), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", fkSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", fkDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "5"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), ), }, { - Config: testConfig(largeSrc, largeDst), + Config: testConfigOfSourceFile(largeSrc, largeDst, 5, 5), Check: resource.ComposeAggregateTestCheckFunc( resource.TestCheckResourceAttr(resourceName, "source.address", largeSrc), resource.TestCheckResourceAttr(resourceName, "destination.address", largeDst), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_open", "5"), + resource.TestCheckResourceAttr(resourceName, "destination.conn_max_idle", "5"), ), }, }, }) } -func testConfig(src, dst string) string { +func testConfigOfSourceFile(src, dst string, dstMaxOpen, dstMaxIdle int) string { const tmpl = ` resource "byteset_pipeline" "test" { source = { @@ -198,8 +216,14 @@ resource "byteset_pipeline" "test" { } destination = { address = "{{ .Dst }}" + conn_max_open = {{ .DstMaxOpen }} + conn_max_idle = {{ .DstMaxIdle }} } }` - return renderConfigTemplate(tmpl, "Src", src, "Dst", dst) + return renderConfigTemplate(tmpl, + "Src", src, + "Dst", dst, + "DstMaxOpen", dstMaxOpen, + "DstMaxIdle", dstMaxIdle) } diff --git a/byteset/testdata/mysql-fk.sql b/byteset/testdata/mysql-fk.sql index 1638958..94b597e 100644 --- a/byteset/testdata/mysql-fk.sql +++ b/byteset/testdata/mysql-fk.sql @@ -21,8 +21,6 @@ CREATE TABLE members ); -- members data -;; -;; INSERT INTO members (id, last_name, first_name, team_id) VALUES (1, 'Lucy', 'Li', 1); INSERT INTO members (id, last_name, first_name, team_id) @@ -33,8 +31,6 @@ INSERT INTO members (id, last_name, first_name, team_id) VALUES (4, 'Frank', NULL, 2); -- teams data -;; -;; INSERT INTO teams (id, name) VALUES (1, 'Finance'); INSERT INTO teams (id, name) diff --git a/byteset/testdata/mysql.sql b/byteset/testdata/mysql.sql index 6cdcf18..2b0465f 100644 --- a/byteset/testdata/mysql.sql +++ b/byteset/testdata/mysql.sql @@ -7,12 +7,10 @@ CREATE TABLE company age INT NOT NULL, address CHAR(50), salary NUMERIC -);; +); -- company data -;; -;; INSERT INTO company (name, age, address, salary) VALUES ('Paul', 32, 'California', 20000.00); INSERT INTO company (name, age, address, salary) diff --git a/byteset/testdata/postgres-fk.sql b/byteset/testdata/postgres-fk.sql index a0fb2fd..0f178e9 100644 --- a/byteset/testdata/postgres-fk.sql +++ b/byteset/testdata/postgres-fk.sql @@ -22,8 +22,6 @@ CREATE TABLE members -- members data -;; -;; INSERT INTO members (id, last_name, first_name, team_id) VALUES (1, 'Lucy', 'Li', 1); INSERT INTO members (id, last_name, first_name, team_id) @@ -34,8 +32,6 @@ INSERT INTO members (id, last_name, first_name, team_id) VALUES (4, 'Frank', NULL, 2); -- teams data -;; -;; INSERT INTO teams (id, name) VALUES (1, 'Finance'); INSERT INTO teams (id, name) diff --git a/byteset/testdata/postgres.sql b/byteset/testdata/postgres.sql index 76b1e7e..25148bf 100644 --- a/byteset/testdata/postgres.sql +++ b/byteset/testdata/postgres.sql @@ -7,12 +7,10 @@ CREATE TABLE company age INT NOT NULL, address CHAR(50), salary REAL -);; +); -- company data -;; -;; INSERT INTO company (name, age, address, salary) VALUES ('Paul', 32, 'California', 20000.00); INSERT INTO company (name, age, address, salary) diff --git a/byteset/testdata/sqlite-fk.sql b/byteset/testdata/sqlite-fk.sql index 77d07b2..0d70105 100644 --- a/byteset/testdata/sqlite-fk.sql +++ b/byteset/testdata/sqlite-fk.sql @@ -21,8 +21,6 @@ CREATE TABLE members ); -- members data -;; -;; INSERT INTO members (id, last_name, first_name, team_id) VALUES (1, 'Lucy', 'Li', 1); INSERT INTO members (id, last_name, first_name, team_id) @@ -33,8 +31,6 @@ INSERT INTO members (id, last_name, first_name, team_id) VALUES (4, 'Frank', NULL, 2); -- teams data -;; -;; INSERT INTO teams (id, name) VALUES (1, 'Finance'); INSERT INTO teams (id, name) diff --git a/byteset/testdata/sqlite.sql b/byteset/testdata/sqlite.sql index 6bc5060..2f5a923 100644 --- a/byteset/testdata/sqlite.sql +++ b/byteset/testdata/sqlite.sql @@ -11,8 +11,6 @@ CREATE TABLE company -- company data -;; -;; INSERT INTO company (name, age, address, salary) VALUES ('Paul', 32, 'California', 20000.00); INSERT INTO company (name, age, address, salary) diff --git a/docs/index.md b/docs/index.md index db1a883..0d75c4e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -36,7 +36,9 @@ resource "byteset_pipeline" "local_file_to_sqlite" { # Destination to load the SQLite SQL file. destination = { - address = "sqlite:///path/to/sqlite.db?_pragma=foreign_keys(1)" + address = "sqlite:///path/to/sqlite.db?_pragma=foreign_keys(1)" + conn_max_open = 1 + conn_max_idle = 1 } } ``` diff --git a/docs/resources/pipeline.md b/docs/resources/pipeline.md index 54ad402..522a6af 100644 --- a/docs/resources/pipeline.md +++ b/docs/resources/pipeline.md @@ -54,9 +54,10 @@ Required: Optional: -- `conn_max_idle` (Number) The maximum idling connections of destination database. -- `conn_max_life` (Number) The maximum lifetime in seconds of destination database. -- `conn_max_open` (Number) The maximum opening connectors of destination database. +- `conn_max_idle` (Number) The maximum idling connections of destination database, +if the given SQL file is using single transaction, should turn down the "conn_max_idle" to 1. +- `conn_max_open` (Number) The maximum opening connectors of destination database, +if the given SQL file is using single transaction, should turn down the "conn_max_open" to 1. - `salt` (String) The salt assist calculating the destination database has changed but the address not, like the database Terraform Managed Resource ID. diff --git a/examples/sqlite/main.tf b/examples/sqlite/main.tf index c3db1c2..9e7fe64 100644 --- a/examples/sqlite/main.tf +++ b/examples/sqlite/main.tf @@ -16,6 +16,8 @@ resource "byteset_pipeline" "local_file_to_sqlite" { # Destination to load the SQLite SQL file. destination = { - address = "sqlite:///path/to/sqlite.db?_pragma=foreign_keys(1)" + address = "sqlite:///path/to/sqlite.db?_pragma=foreign_keys(1)" + conn_max_open = 1 + conn_max_idle = 1 } } diff --git a/go.mod b/go.mod index e05ccef..c512a7e 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,14 @@ require ( github.com/docker/go-connections v0.4.0 github.com/go-sql-driver/mysql v1.7.1 github.com/hashicorp/terraform-plugin-framework v1.2.0 + github.com/hashicorp/terraform-plugin-framework-validators v0.10.0 github.com/hashicorp/terraform-plugin-go v0.14.3 github.com/hashicorp/terraform-plugin-log v0.8.0 github.com/hashicorp/terraform-plugin-testing v1.2.0 github.com/lib/pq v1.10.9 github.com/sijms/go-ora/v2 v2.7.6 - github.com/stretchr/testify v1.7.2 + github.com/sourcegraph/conc v0.3.0 + github.com/stretchr/testify v1.8.1 golang.org/x/mod v0.10.0 modernc.org/sqlite v1.23.0 ) @@ -52,7 +54,6 @@ require ( github.com/hashicorp/terraform-svchost v0.0.0-20200729002733-f050f53b9734 // indirect github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect @@ -73,6 +74,8 @@ require ( github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect github.com/zclconf/go-cty v1.13.1 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect diff --git a/go.sum b/go.sum index 72936e0..e7ed4e6 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/hashicorp/terraform-json v0.16.0 h1:UKkeWRWb23do5LNAFlh/K3N0ymn1qTOO8 github.com/hashicorp/terraform-json v0.16.0/go.mod h1:v0Ufk9jJnk6tcIZvScHvetlKfiNTC+WS21mnXIlc0B0= github.com/hashicorp/terraform-plugin-framework v1.2.0 h1:MZjFFfULnFq8fh04FqrKPcJ/nGpHOvX4buIygT3MSNY= github.com/hashicorp/terraform-plugin-framework v1.2.0/go.mod h1:nToI62JylqXDq84weLJ/U3umUsBhZAaTmU0HXIVUOcw= +github.com/hashicorp/terraform-plugin-framework-validators v0.10.0 h1:4L0tmy/8esP6OcvocVymw52lY0HyQ5OxB7VNl7k4bS0= +github.com/hashicorp/terraform-plugin-framework-validators v0.10.0/go.mod h1:qdQJCdimB9JeX2YwOpItEu+IrfoJjWQ5PhLpAOMDQAE= github.com/hashicorp/terraform-plugin-go v0.14.3 h1:nlnJ1GXKdMwsC8g1Nh05tK2wsC3+3BL/DBBxFEki+j0= github.com/hashicorp/terraform-plugin-go v0.14.3/go.mod h1:7ees7DMZ263q8wQ6E4RdIdR6nHHJtrdt4ogX5lPkX1A= github.com/hashicorp/terraform-plugin-log v0.8.0 h1:pX2VQ/TGKu+UU1rCay0OlzosNKe4Nz1pepLXj95oyy0= @@ -166,7 +168,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -226,24 +227,31 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sijms/go-ora/v2 v2.7.6 h1:QyR1CKFxG+VVk2+LdHoHF4NxDSvcQ3deBXtZCrahSq4= github.com/sijms/go-ora/v2 v2.7.6/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= @@ -260,6 +268,10 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= github.com/zclconf/go-cty v1.13.1 h1:0a6bRwuiSHtAmqCqNOE+c2oHgepv0ctoxU4FUe43kwc= github.com/zclconf/go-cty v1.13.1/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -404,7 +416,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index d5074d0..ac7e9d0 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,13 @@ package main import ( - "context" "flag" "log" "github.com/hashicorp/terraform-plugin-framework/providerserver" "github.com/seal-io/terraform-provider-byteset/byteset" + "github.com/seal-io/terraform-provider-byteset/utils/signalx" ) func main() { @@ -22,7 +22,7 @@ func main() { flag.Parse() err := providerserver.Serve( - context.Background(), + signalx.Context(), byteset.NewProvider, providerserver.ServeOpts{ Address: byteset.ProviderAddress, diff --git a/pipeline/destination.go b/pipeline/destination.go index cab726f..8947496 100644 --- a/pipeline/destination.go +++ b/pipeline/destination.go @@ -7,13 +7,17 @@ import ( "io" "time" + "github.com/sourcegraph/conc/pool" + "github.com/seal-io/terraform-provider-byteset/utils/sqlx" ) type Destination interface { io.Closer - Exec(ctx context.Context, query string, args ...any) error + // Exec executes the given statement with arguments, + // detects the given statement and pick the proper execution(sync/async) to finish the job. + Exec(ctx context.Context, statement string, args ...any) error } func NewDestination(ctx context.Context, addr string, opts ...Option) (Destination, error) { @@ -22,6 +26,17 @@ func NewDestination(ctx context.Context, addr string, opts ...Option) (Destinati return nil, fmt.Errorf("cannot load database from %q: %w", addr, err) } + // Configure. + opts = append(opts, WithConnMaxLife(0)) + for i := range opts { + if opts[i] == nil { + continue + } + + opts[i](db) + } + + // Detect connectivity. ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -30,27 +45,118 @@ func NewDestination(ctx context.Context, addr string, opts ...Option) (Destinati return nil, fmt.Errorf("cannot connect database on %q: %w", addr, err) } - for i := range opts { - if opts[i] == nil { - continue - } + return &dst{drv: drv, db: db, dbStats: db.Stats()}, nil +} - opts[i](db) - } +type dst struct { + drv string + db *sql.DB + dbStats sql.DBStats - return dst{drv: drv, db: db}, nil + gp *pool.ContextPool } -type dst struct { - drv string - db *sql.DB +func (in *dst) Close() error { + return in.db.Close() } -func (d dst) Close() error { - return d.db.Close() +func (in *dst) Exec(ctx context.Context, statement string, args ...any) (err error) { + if in.dbStats.MaxOpenConnections == 1 { + err = in.exec(ctx, statement, args...) + } else { + err = in.execAsync(ctx, statement, args...) + } + + if err != nil { + return fmt.Errorf("cannot execute %q: %w", statement, err) + } + + return } -func (d dst) Exec(ctx context.Context, query string, args ...any) error { - _, err := d.db.ExecContext(ctx, query, args...) - return err +func (in *dst) exec(ctx context.Context, statement string, args ...any) (err error) { + _, err = in.db.ExecContext(ctx, statement, args...) + if err != nil { + if sqlx.IsEmptyError(err) { + err = nil + } + } + + return +} + +func (in *dst) execAsync(ctx context.Context, statement string, args ...any) error { + if sqlx.IsDCL(statement) { + if in.gp != nil { + // Wait for all previous DDL finishing. + if err := in.gp.Wait(); err != nil { + return err + } + in.gp = nil + } + + // Switch to sync mode. + in.db.SetMaxIdleConns(1) + in.db.SetMaxOpenConns(1) + in.dbStats = in.db.Stats() + + return in.exec(ctx, statement, args...) + } + + if sqlx.IsDML(statement) { + // Execute DML in async mode. + if in.gp == nil { + in.gp = pool.New(). + WithMaxGoroutines(in.dbStats.MaxOpenConnections). + WithContext(ctx). + WithFirstError() + } + + in.gp.Go(func(ctx context.Context) error { + return in.exec(ctx, statement, args...) + }) + + return nil + } + + if in.gp != nil { + // Wait for all previous DDL finishing. + if err := in.gp.Wait(); err != nil { + return err + } + in.gp = nil + } + + if sqlx.IsDDL(statement) { + // Execute DDL in one connection. + return in.exec(ctx, statement, args...) + } + + // Execute DCL for all connections. + cs := make([]*sql.Conn, 0, in.dbStats.MaxOpenConnections) + defer func() { + for i := range cs { + _ = cs[i].Close() + } + }() + + for i := 0; i < in.dbStats.MaxOpenConnections; i++ { + c, err := in.db.Conn(ctx) + if err != nil { + return err + } + + cs = append(cs, c) + + _, err = c.ExecContext(ctx, statement, args...) + if err != nil { + if sqlx.IsEmptyError(err) { + continue + } + + return err + } + } + + return nil } diff --git a/pipeline/option.go b/pipeline/option.go index 16e92c0..db9810c 100644 --- a/pipeline/option.go +++ b/pipeline/option.go @@ -10,7 +10,7 @@ type Option func(db *sql.DB) func WithConnMaxOpen(o int) Option { return func(db *sql.DB) { if o <= 0 { - return + o = 5 } db.SetMaxOpenConns(o) @@ -19,6 +19,10 @@ func WithConnMaxOpen(o int) Option { func WithConnMaxIdle(i int) Option { return func(db *sql.DB) { + if i <= 0 { + i = 5 + } + db.SetMaxIdleConns(i) } } diff --git a/pipeline/source.go b/pipeline/source.go index a8f984c..08aa548 100644 --- a/pipeline/source.go +++ b/pipeline/source.go @@ -2,7 +2,6 @@ package pipeline import ( "bufio" - "bytes" "context" "database/sql" "fmt" @@ -33,7 +32,7 @@ func NewSource(ctx context.Context, addr string, opts ...Option) (Source, error) return nil, fmt.Errorf("cannot open local file from %q: %w", addr, err) } - return srcFile{f: local}, nil + return &srcFile{f: local}, nil case strings.HasPrefix(addr, "http://") || strings.HasPrefix(addr, "https://"): remote, err := http.Get(addr) @@ -41,7 +40,7 @@ func NewSource(ctx context.Context, addr string, opts ...Option) (Source, error) return nil, fmt.Errorf("cannot open remote file from %q: %w", addr, err) } - return srcFile{f: remote.Body}, nil + return &srcFile{f: remote.Body}, nil default: } @@ -51,14 +50,7 @@ func NewSource(ctx context.Context, addr string, opts ...Option) (Source, error) return nil, fmt.Errorf("cannot load database from %q: %w", addr, err) } - ctx, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - - err = sqlx.IsDatabaseConnected(ctx, db) - if err != nil { - return nil, fmt.Errorf("cannot connect database on %q: %w", addr, err) - } - + // Configure. for i := range opts { if opts[i] == nil { continue @@ -67,70 +59,39 @@ func NewSource(ctx context.Context, addr string, opts ...Option) (Source, error) opts[i](db) } - return srcDatabase{drv: drv, db: db}, nil + // Detect connectivity. + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + err = sqlx.IsDatabaseConnected(ctx, db) + if err != nil { + return nil, fmt.Errorf("cannot connect database on %q: %w", addr, err) + } + + return &srcDatabase{drv: drv, db: db}, nil } type srcFile struct { f io.ReadCloser } -func (s srcFile) Close() error { - return s.f.Close() +func (in *srcFile) Close() error { + return in.f.Close() } -func (s srcFile) Pipe(ctx context.Context, dst Destination) error { - line := func(data []byte, eof bool) (int, []byte, error) { - if eof && len(data) == 0 { - return 0, nil, nil - } - - var ( - i int - d = data - ) +func (in *srcFile) Pipe(ctx context.Context, dst Destination) error { + ss := bufio.NewScanner(in.f) + ss.Split(split) - for { - if j := bytes.IndexByte(d, '\n'); j >= 0 { - if (j == 1 && d[j-1] == ';') || - (j > 1 && (d[j-1] == ';' || d[j-2] == ';')) { - return i + j + 1, bytes.TrimLeft(bytes.TrimRight(data[0:i+j], "\r"), "\r\n"), nil - } + for ss.Scan() { + s := ss.Text() - if j+1 >= len(d) { - break - } - d = d[j+1:] - i += j + 1 - - continue - } - - break - } - - if eof { - return len(data), bytes.TrimLeft(bytes.TrimRight(data, "\r"), "\r\n"), nil - } - - return 0, nil, nil - } - - qs := bufio.NewScanner(s.f) - qs.Split(line) - - for qs.Scan() { - q := qs.Text() - - err := dst.Exec(ctx, q) + err := dst.Exec(ctx, s) if err != nil { - if !isEmptyQueryError(err) { - return fmt.Errorf("cannot execute %q: %w", q, err) - } - - continue + return err } - tflog.Debug(ctx, "Executed", map[string]any{"query": q}) + tflog.Debug(ctx, "Executed", map[string]any{"statement": s}) } return nil @@ -141,22 +102,10 @@ type srcDatabase struct { db *sql.DB } -func (s srcDatabase) Close() error { - return s.db.Close() +func (in *srcDatabase) Close() error { + return in.db.Close() } -func (s srcDatabase) Pipe(ctx context.Context, dst Destination) error { +func (in *srcDatabase) Pipe(ctx context.Context, dst Destination) error { return nil } - -func isEmptyQueryError(err error) bool { - for _, s := range []string{ - "Error 1065", // MySQL (Query was empty). - } { - if strings.Contains(err.Error(), s) { - return true - } - } - - return false -} diff --git a/pipeline/source_file.go b/pipeline/source_file.go new file mode 100644 index 0000000..8f88688 --- /dev/null +++ b/pipeline/source_file.go @@ -0,0 +1,86 @@ +package pipeline + +import "bytes" + +func split(data []byte, atEOF bool) (int, []byte, error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + var ( + ds int + dlt = dropStartCRLF(data) + dll = len(data) - len(dlt) + ) + + dataRst := data + + for { + if de := bytes.IndexByte(dataRst, '\n'); de >= 0 { + if de >= 1 { + di := ds + de + d := data[ds:di] + drt := dropEndCR(d) + + drl := len(d) - len(drt) + + switch { + case de > 1 && dlt[0] == '-' && dlt[1] == '-': // Start with '--'. + return di + 1, data[dll : di-drl], nil + case de > 1 && dlt[0] == '/' && dlt[1] == '*': // Start with '/*'. + // End with '*/'. + if drt[de-drl-2] == '*' && drt[de-drl-1] == '/' { + return di + 1, data[dll : di-drl], nil + } + + // Revert searching. + var ending bool + + for i := len(drt) - 1; i >= 0; i-- { + if drt[i] == '/' { + ending = true + continue + } + + if ending { + if drt[i] == '*' { + // Comment command. + return di + 1, data[dll : di-drl], nil + } + + break + } + } + case drt[de-drl-1] == ';': // End with ';'. + return di + 1, data[dll : di-drl], nil + } + } + + if de+1 >= len(dataRst) { + break + } + dataRst = dataRst[de+1:] + ds += de + 1 + + continue + } + + break + } + + if atEOF { + return len(data), dropEndCR(data[dll:]), nil + } + + return 0, nil, nil +} + +// dropStartCRLF drops leading \r\n from the data. +func dropStartCRLF(data []byte) []byte { + return bytes.TrimLeft(data, "\r\n") +} + +// dropEndCR drops terminal \r from the data. +func dropEndCR(data []byte) []byte { + return bytes.TrimRight(data, "\r") +} diff --git a/pipeline/source_file_test.go b/pipeline/source_file_test.go new file mode 100644 index 0000000..7f8988e --- /dev/null +++ b/pipeline/source_file_test.go @@ -0,0 +1,80 @@ +package pipeline + +import ( + "bufio" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/seal-io/terraform-provider-byteset/utils/testx" +) + +func TestSource_srcFile_split(t *testing.T) { + f, err := testx.File("testdata/complex.sql") + if err != nil { + panic(err) + } + + defer func() { _ = f.Close() }() + + var actual []string + ss := bufio.NewScanner(f) + ss.Split(split) + + for ss.Scan() { + actual = append(actual, ss.Text()) + } + + expected := []string{ + `-- Comment 1`, + + `--`, + `-- Comment 2`, + `--`, + + `-- /// Comment 3`, + + `-- /* Comment 4 */`, + + `/* -- Comment 5 */`, + + `/* Comment 6 */`, + + `/* Comment 7 */;`, + + `/* + Comment 8 + */;`, + + `/* + Comment 9 + */ ;`, + + `/* + Comment 10; +*/`, + + `;`, + + `;;`, + + `;;;`, + + `/*!40014 SET @OLD_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS = 0 */;`, + + `DROP TABLE IF EXISTS test;`, + + `CREATE TABLE test +( + id INTEGER PRIMARY KEY AUTO_INCREMENT, + val REAL +);`, + + `INSERT INTO test (val) +VALUES ('Test 1');`, + + `INSERT INTO test (val) VALUES ('Test 2');`, + } + + assert.Equal(t, expected, actual) +} diff --git a/pipeline/testdata/complex.sql b/pipeline/testdata/complex.sql new file mode 100644 index 0000000..594d56e --- /dev/null +++ b/pipeline/testdata/complex.sql @@ -0,0 +1,47 @@ + + +-- Comment 1 + +-- +-- Comment 2 +-- + +-- /// Comment 3 + +-- /* Comment 4 */ + +/* -- Comment 5 */ + +/* Comment 6 */ + +/* Comment 7 */; + +/* + Comment 8 + */; + +/* + Comment 9 + */ ; + +/* + Comment 10; +*/ + +; +;; +;;; + +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS = 0 */; + +DROP TABLE IF EXISTS test; +CREATE TABLE test +( + id INTEGER PRIMARY KEY AUTO_INCREMENT, + val REAL +); + +INSERT INTO test (val) +VALUES ('Test 1'); + +INSERT INTO test (val) VALUES ('Test 2'); diff --git a/utils/signalx/doc.go b/utils/signalx/doc.go new file mode 100644 index 0000000..fa3983b --- /dev/null +++ b/utils/signalx/doc.go @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: 2017 Kubernetes. +// SPDX-License-Identifier: Apache-2.0 + +// Package signalx contains libraries for handling signals to gracefully +// shut down the manager in combination with Kubernetes pod graceful termination +// policy. +package signalx diff --git a/utils/signalx/signal.go b/utils/signalx/signal.go new file mode 100644 index 0000000..88ec0dc --- /dev/null +++ b/utils/signalx/signal.go @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2017 Kubernetes. +// SPDX-License-Identifier: Apache-2.0 + +package signalx + +import ( + "context" + "os" + "os/signal" +) + +var onlyOneSignalHandler = make(chan struct{}) + +// Context registers for SIGTERM and SIGINT. +// A context is returned which is canceled on one of these signals. +// If a second signal is caught, the program is terminated with exit code 1. +func Context() context.Context { + close(onlyOneSignalHandler) // Panics when called twice. + + ctx, cancel := context.WithCancel(context.Background()) + + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + + go func() { + <-c + cancel() + <-c + os.Exit(1) // Second signal. Exit directly. + }() + + return ctx +} diff --git a/utils/signalx/signal_posix.go b/utils/signalx/signal_posix.go new file mode 100644 index 0000000..747766c --- /dev/null +++ b/utils/signalx/signal_posix.go @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2017 Kubernetes. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !windows + +package signalx + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/utils/signalx/signal_windows.go b/utils/signalx/signal_windows.go new file mode 100644 index 0000000..02aa59f --- /dev/null +++ b/utils/signalx/signal_windows.go @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2017 Kubernetes. +// SPDX-License-Identifier: Apache-2.0 + +package signalx + +import ( + "os" +) + +var shutdownSignals = []os.Signal{os.Interrupt} diff --git a/utils/sqlx/driver.go b/utils/sqlx/driver.go index 253b2e4..68046c8 100644 --- a/utils/sqlx/driver.go +++ b/utils/sqlx/driver.go @@ -57,7 +57,13 @@ func LoadDatabase(addr string) (drv string, db *sql.DB, err error) { if err != nil { return } + db, err = sql.Open(drv, dsn) + if err != nil { + return + } + + db.SetConnMaxIdleTime(0) return } diff --git a/utils/sqlx/helper.go b/utils/sqlx/helper.go new file mode 100644 index 0000000..521e5ba --- /dev/null +++ b/utils/sqlx/helper.go @@ -0,0 +1,76 @@ +package sqlx + +import ( + "strings" +) + +var dmlPrefixes = []string{ + "INSERT ", + "DELETE ", + "UPDATE ", +} + +func IsDML(s string) bool { + if s != "" { + for i := range dmlPrefixes { + if strings.HasPrefix(s, dmlPrefixes[i]) { + return true + } + } + } + + return false +} + +var ddlPrefixes = []string{ + "DROP ", + "ALTER ", + "CREATE ", +} + +func IsDDL(s string) bool { + if s != "" { + for i := range ddlPrefixes { + if strings.HasPrefix(s, ddlPrefixes[i]) { + return true + } + } + } + + return false +} + +var dclPrefixes = []string{ + "LOCK ", "UNLOCK ", // MySQL. + "COPY ", // Postgres. +} + +func IsDCL(s string) bool { + if s != "" { + for i := range dclPrefixes { + if strings.HasPrefix(s, dclPrefixes[i]) { + return true + } + } + } + + return false +} + +var emptyErrMessages = []string{ + "sql: no rows in result set", + "Error 1065", // MySQL (Query was empty). +} + +func IsEmptyError(err error) bool { + if err != nil { + m := err.Error() + for i := range emptyErrMessages { + if strings.Contains(m, emptyErrMessages[i]) { + return true + } + } + } + + return false +} diff --git a/utils/testx/file.go b/utils/testx/file.go new file mode 100644 index 0000000..3532080 --- /dev/null +++ b/utils/testx/file.go @@ -0,0 +1,19 @@ +package testx + +import ( + "os" + "path/filepath" +) + +func AbsolutePath(relativePath string) string { + dir, err := os.Getwd() + if err != nil { + panic(err) + } + + return filepath.Join(dir, relativePath) +} + +func File(relativePath string) (*os.File, error) { + return os.Open(AbsolutePath(relativePath)) +}