Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Fully parallelize query execution #28

Draft
wants to merge 11 commits into
base: main
Choose a base branch
from
6 changes: 3 additions & 3 deletions cmd/pgverify/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"fmt"
"strings"

"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"

Expand Down Expand Up @@ -49,9 +49,9 @@ var rootCmd = &cobra.Command{
Long: `Verify data consistency between PostgreSQL syntax compatible databases.`,
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
var targets []*pgx.ConnConfig
var targets []*pgxpool.Config
for _, target := range args {
connConfig, err := pgx.ParseConfig(target)
connConfig, err := pgxpool.ParseConfig(target)
if err != nil {
return fmt.Errorf("invalid target URI %s: %w", target, err)
}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ require (
github.com/jackc/pgproto3/v2 v2.2.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgtype v1.10.0 // indirect
github.com/jackc/puddle v1.2.1 // indirect
github.com/jgautheron/goconst v1.6.0 // indirect
github.com/jingyugao/rowserrcheck v1.1.1 // indirect
github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,7 @@ github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.2.1 h1:gI8os0wpRXFd4FiAY2dWiqRK037tjj3t7rKFeO4X5iw=
github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jgautheron/goconst v1.6.0 h1:gbMLWKRMkzAc6kYsQL6/TxaoBUg3Jm9LSF/Ih1ADWGA=
github.com/jgautheron/goconst v1.6.0/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4=
Expand Down
16 changes: 8 additions & 8 deletions integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (

"github.com/cjfinnell/pgverify"
"github.com/google/uuid"
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -27,17 +27,17 @@ var (
dbName = "test"
)

func waitForDBReady(t *testing.T, ctx context.Context, config *pgx.ConnConfig) bool {
func waitForDBReady(t *testing.T, ctx context.Context, config *pgxpool.Config) bool {
t.Helper()

connected := false

for count := 0; count < 30; count++ {
conn, err := pgx.ConnectConfig(ctx, config)
conn, err := pgxpool.ConnectConfig(ctx, config)
if err == nil {
connected = true

conn.Close(ctx)
conn.Close()

break
}
Expand Down Expand Up @@ -257,7 +257,7 @@ func TestVerifyData(t *testing.T) {
}

// Act
var targets []*pgx.ConnConfig
var targets []*pgxpool.Config

var aliases []string

Expand All @@ -266,13 +266,13 @@ func TestVerifyData(t *testing.T) {
// Create db and connect
_, port, err := createContainer(t, ctx, db.image, db.port, db.env, db.cmd)
require.NoError(t, err)
config, err := pgx.ParseConfig(fmt.Sprintf("postgresql://%[email protected]:%d%s", db.userPassword, port, db.db))
config, err := pgxpool.ParseConfig(fmt.Sprintf("postgresql://%[email protected]:%d%s", db.userPassword, port, db.db))
require.NoError(t, err)
assert.True(t, waitForDBReady(t, ctx, config))
conn, err := pgx.ConnectConfig(ctx, config)
conn, err := pgxpool.ConnectConfig(ctx, config)
require.NoError(t, err)

defer conn.Close(ctx)
defer conn.Close()

// Create and populate tables
for _, tableName := range tableNames {
Expand Down
18 changes: 13 additions & 5 deletions results.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,24 @@ func NewResults(targetNames []string, testModes []string) *Results {
}
}

// SingleResult represents the verification result from a single target, with the schema:
// SingleResult[schema][table][mode] = test output.
type SingleResult map[string]map[string]map[string]string
// DatabaseResult represents the verification result from a single target database:
// DatabaseResult[schema][table][mode] = test output.
type DatabaseResult map[string]SchemaResult

// SchemaResult represents the verification result from a single schema:
// SchemaResult[table][mode] = test output.
type SchemaResult map[string]TableResult

// TableResult represents the verification result from a single table:
// TableResult[mode] = test output.
type TableResult map[string]string

// AddResult adds a SingleResult from a test on a specific target to the Results object.
func (r *Results) AddResult(targetName string, schemaTableHashes SingleResult) {
func (r *Results) AddResult(targetName string, databaseHashes DatabaseResult) {
r.mutex.Lock()
defer r.mutex.Unlock()

for schema, tables := range schemaTableHashes {
for schema, tables := range databaseHashes {
if _, ok := r.content[schema]; !ok {
r.content[schema] = make(map[string]map[string]map[string][]string)
}
Expand Down
60 changes: 60 additions & 0 deletions vendor/github.com/jackc/pgx/v4/pgxpool/batch_results.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

109 changes: 109 additions & 0 deletions vendor/github.com/jackc/pgx/v4/pgxpool/conn.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 25 additions & 0 deletions vendor/github.com/jackc/pgx/v4/pgxpool/doc.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading