ci: run database tests against Postgres, MySQL and SQLite (#6996)

.github/workflows/go.yml

@@ -24,7 +24,7 @@ permissions:
 jobs:
   lint:
     permissions:
-      contents: read  # for actions/checkout to fetch code
+      contents: read       # for actions/checkout to fetch code
       pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
     name: Lint
     runs-on: ubuntu-latest
@@ -88,3 +88,64 @@ jobs:
             The job "${{ github.job }}" of ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} completed with "${{ job.status }}".
 
             View the job run at: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+  postgres:
+    name: Postgres
+    strategy:
+      matrix:
+        go-version: [ 1.16.x, 1.17.x, 1.18.x ]
+        platform: [ ubuntu-latest ]
+    runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: postgres:9.6
+        env:
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run tests with coverage
+        run: go test -v -race -coverprofile=coverage -covermode=atomic ./internal/db
+        env:
+          GOGS_DATABASE_TYPE: postgres
+          PGPORT: 5432
+          PGHOST: localhost
+          PGUSER: postgres
+          PGPASSWORD: postgres
+          PGSSLMODE: disable
+
+  mysql:
+    name: MySQL
+    strategy:
+      matrix:
+        go-version: [ 1.16.x, 1.17.x, 1.18.x ]
+        platform: [ ubuntu-18.04 ]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Start MySQL server
+        run: sudo systemctl start mysql
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run tests with coverage
+        run: go test -v -race -coverprofile=coverage -covermode=atomic ./internal/db
+        env:
+          GOGS_DATABASE_TYPE: mysql
+          MYSQL_USER: root
+          MYSQL_PASSWORD: root
+          MYSQL_HOST: localhost
+          MYSQL_PORT: 3306

.gitignore

@@ -15,3 +15,4 @@ profile/
 output*
 /release
 .task
+.envrc

go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/urfave/cli v1.22.9
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/text v0.3.7
 	gopkg.in/DATA-DOG/go-sqlmock.v2 v2.0.0-20180914054222-c19298f520d0
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

go.sum

@@ -599,8 +599,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

internal/db/backup.go

@@ -85,6 +85,11 @@ func dumpTable(db *gorm.DB, table interface{}, w io.Writer) error {
 			return errors.Wrap(err, "scan rows")
 		}
 
+		switch e := elem.(type) {
+		case *LFSObject:
+			e.CreatedAt = e.CreatedAt.UTC()
+		}
+
 		err = jsoniter.NewEncoder(w).Encode(elem)
 		if err != nil {
 			return errors.Wrap(err, "encode JSON")

internal/db/db.go

@@ -53,8 +53,8 @@ func parseMSSQLHostPort(info string) (host, port string) {
 	return host, port
 }
 
-// parseDSN takes given database options and returns parsed DSN.
+// newDSN takes given database options and returns parsed DSN.
-func parseDSN(opts conf.DatabaseOpts) (dsn string, err error) {
+func newDSN(opts conf.DatabaseOpts) (dsn string, err error) {
 	// In case the database name contains "?" with some parameters
 	concate := "?"
 	if strings.Contains(opts.Name, concate) {
@@ -109,7 +109,7 @@ func newLogWriter() (logger.Writer, error) {
 }
 
 func openDB(opts conf.DatabaseOpts, cfg *gorm.Config) (*gorm.DB, error) {
-	dsn, err := parseDSN(opts)
+	dsn, err := newDSN(opts)
 	if err != nil {
 		return nil, errors.Wrap(err, "parse DSN")
 	}
@@ -151,14 +151,18 @@ func Init(w logger.Writer) (*gorm.DB, error) {
 		LogLevel:      level,
 	})
 
-	db, err := openDB(conf.Database, &gorm.Config{
+	db, err := openDB(
-		NamingStrategy: schema.NamingStrategy{
+		conf.Database,
-			SingularTable: true,
+		&gorm.Config{
+			SkipDefaultTransaction: true,
+			NamingStrategy: schema.NamingStrategy{
+				SingularTable: true,
+			},
+			NowFunc: func() time.Time {
+				return time.Now().UTC().Truncate(time.Microsecond)
+			},
 		},
-		NowFunc: func() time.Time {
+	)
-			return time.Now().UTC().Truncate(time.Microsecond)
-		},
-	})
 	if err != nil {
 		return nil, errors.Wrap(err, "open database")
 	}

internal/db/db_test.go

@@ -56,7 +56,7 @@ func Test_parseMSSQLHostPort(t *testing.T) {
 
 func Test_parseDSN(t *testing.T) {
 	t.Run("bad dialect", func(t *testing.T) {
-		_, err := parseDSN(conf.DatabaseOpts{
+		_, err := newDSN(conf.DatabaseOpts{
 			Type: "bad_dialect",
 		})
 		assert.Equal(t, "unrecognized dialect: bad_dialect", fmt.Sprintf("%v", err))
@@ -140,7 +140,7 @@ func Test_parseDSN(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			dsn, err := parseDSN(test.opts)
+			dsn, err := newDSN(test.opts)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/db/lfs_test.go

@@ -43,6 +43,9 @@ func Test_lfs(t *testing.T) {
 			})
 			tc.test(t, db)
 		})
+		if t.Failed() {
+			break
+		}
 	}
 }
 
@@ -60,7 +63,7 @@ func test_lfs_CreateObject(t *testing.T, db *lfs) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	assert.Equal(t, db.NowFunc().Format(time.RFC3339), object.CreatedAt.Format(time.RFC3339))
+	assert.Equal(t, db.NowFunc().Format(time.RFC3339), object.CreatedAt.UTC().Format(time.RFC3339))
 
 	// Try create second LFS object with same oid should fail
 	err = db.CreateObject(repoID, oid, 12, lfsutil.StorageLocal)

internal/db/login_sources_test.go

@@ -21,6 +21,7 @@ func TestLoginSource_BeforeSave(t *testing.T) {
 	now := time.Now()
 	db := &gorm.DB{
 		Config: &gorm.Config{
+			SkipDefaultTransaction: true,
 			NowFunc: func() time.Time {
 				return now
 			},
@@ -54,6 +55,7 @@ func TestLoginSource_BeforeCreate(t *testing.T) {
 	now := time.Now()
 	db := &gorm.DB{
 		Config: &gorm.Config{
+			SkipDefaultTransaction: true,
 			NowFunc: func() time.Time {
 				return now
 			},
@@ -108,6 +110,9 @@ func Test_loginSources(t *testing.T) {
 			})
 			tc.test(t, db)
 		})
+		if t.Failed() {
+			break
+		}
 	}
 }
 

internal/db/main_test.go

@@ -5,6 +5,7 @@
 package db
 
 import (
+	"database/sql"
 	"flag"
 	"fmt"
 	"os"
@@ -12,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
 	"gorm.io/gorm"
 	"gorm.io/gorm/logger"
 	"gorm.io/gorm/schema"
@@ -59,16 +61,92 @@ func clearTables(t *testing.T, db *gorm.DB, tables ...interface{}) error {
 }
 
 func initTestDB(t *testing.T, suite string, tables ...interface{}) *gorm.DB {
-	t.Helper()
+	dbType := os.Getenv("GOGS_DATABASE_TYPE")
+
+	var dbName string
+	var dbOpts conf.DatabaseOpts
+	var cleanup func(db *gorm.DB)
+	switch dbType {
+	case "mysql":
+		dbOpts = conf.DatabaseOpts{
+			Type:     "mysql",
+			Host:     os.ExpandEnv("$MYSQL_HOST:$MYSQL_PORT"),
+			Name:     dbName,
+			User:     os.Getenv("MYSQL_USER"),
+			Password: os.Getenv("MYSQL_PASSWORD"),
+		}
+
+		dsn, err := newDSN(dbOpts)
+		require.NoError(t, err)
+
+		sqlDB, err := sql.Open("mysql", dsn)
+		require.NoError(t, err)
+
+		// Set up test database
+		dbName = fmt.Sprintf("gogs-%s-%d", suite, time.Now().Unix())
+		_, err = sqlDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbName))
+		require.NoError(t, err)
+
+		_, err = sqlDB.Exec(fmt.Sprintf("CREATE DATABASE `%s`", dbName))
+		require.NoError(t, err)
+
+		dbOpts.Name = dbName
+
+		cleanup = func(db *gorm.DB) {
+			db.Exec(fmt.Sprintf("DROP DATABASE `%s`", dbName))
+			_ = sqlDB.Close()
+		}
+	case "postgres":
+		dbOpts = conf.DatabaseOpts{
+			Type:     "postgres",
+			Host:     os.ExpandEnv("$PGHOST:$PGPORT"),
+			Name:     dbName,
+			Schema:   "public",
+			User:     os.Getenv("PGUSER"),
+			Password: os.Getenv("PGPASSWORD"),
+			SSLMode:  os.Getenv("PGSSLMODE"),
+		}
+
+		dsn, err := newDSN(dbOpts)
+		require.NoError(t, err)
+
+		sqlDB, err := sql.Open("pgx", dsn)
+		require.NoError(t, err)
+
+		// Set up test database
+		dbName = fmt.Sprintf("gogs-%s-%d", suite, time.Now().Unix())
+		_, err = sqlDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %q", dbName))
+		require.NoError(t, err)
+
+		_, err = sqlDB.Exec(fmt.Sprintf("CREATE DATABASE %q", dbName))
+		require.NoError(t, err)
+
+		dbOpts.Name = dbName
+
+		cleanup = func(db *gorm.DB) {
+			db.Exec(fmt.Sprintf(`DROP DATABASE %q`, dbName))
+			_ = sqlDB.Close()
+		}
+	default:
+		dbName = filepath.Join(os.TempDir(), fmt.Sprintf("gogs-%s-%d.db", suite, time.Now().Unix()))
+		dbOpts = conf.DatabaseOpts{
+			Type: "sqlite3",
+			Path: dbName,
+		}
+		cleanup = func(db *gorm.DB) {
+			sqlDB, err := db.DB()
+			if err == nil {
+				_ = sqlDB.Close()
+			}
+			_ = os.Remove(dbName)
+		}
+	}
 
-	dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("gogs-%s-%d.db", suite, time.Now().Unix()))
 	now := time.Now().UTC().Truncate(time.Second)
 	db, err := openDB(
-		conf.DatabaseOpts{
+		dbOpts,
-			Type: "sqlite3",
-			Path: dbpath,
-		},
 		&gorm.Config{
+			SkipDefaultTransaction: true,
 			NamingStrategy: schema.NamingStrategy{
 				SingularTable: true,
 			},
@@ -77,27 +155,19 @@ func initTestDB(t *testing.T, suite string, tables ...interface{}) *gorm.DB {
 			},
 		},
 	)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
-	t.Cleanup(func() {
-		sqlDB, err := db.DB()
-		if err == nil {
-			_ = sqlDB.Close()
-		}
 
+	t.Cleanup(func() {
 		if t.Failed() {
-			t.Logf("Database %q left intact for inspection", dbpath)
+			t.Logf("Database %q left intact for inspection", dbName)
 			return
 		}
 
-		_ = os.Remove(dbpath)
+		cleanup(db)
 	})
 
 	err = db.Migrator().AutoMigrate(tables...)
-	if err != nil {
+	require.NoError(t, err)
-		t.Fatal(err)
-	}
 
 	return db
 }

internal/db/perms_test.go

@@ -39,6 +39,9 @@ func Test_perms(t *testing.T) {
 			})
 			tc.test(t, db)
 		})
+		if t.Failed() {
+			break
+		}
 	}
 }
 

internal/db/repos_test.go

@@ -41,6 +41,9 @@ func Test_repos(t *testing.T) {
 			})
 			tc.test(t, db)
 		})
+		if t.Failed() {
+			break
+		}
 	}
 }
 

internal/db/two_factors_test.go

@@ -42,6 +42,9 @@ func Test_twoFactors(t *testing.T) {
 			})
 			tc.test(t, db)
 		})
+		if t.Failed() {
+			break
+		}
 	}
 }
 

internal/db/users_test.go

@@ -45,6 +45,9 @@ func Test_users(t *testing.T) {
 			})
 			tc.test(t, db)
 		})
+		if t.Failed() {
+			break
+		}
 	}
 }
 
@@ -136,7 +139,7 @@ func test_users_GetByEmail(t *testing.T, db *users) {
 			t.Fatal(err)
 		}
 
-		err = db.Exec(`UPDATE user SET type = ? WHERE id = ?`, UserOrganization, org.ID).Error
+		err = db.Model(&User{}).Where("id", org.ID).UpdateColumn("type", UserOrganization).Error
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -158,7 +161,7 @@ func test_users_GetByEmail(t *testing.T, db *users) {
 
 		// Mark user as activated
 		// TODO: Use UserEmails.Verify to replace SQL hack when the method is available.
-		err = db.Exec(`UPDATE user SET is_active = ? WHERE id = ?`, true, alice.ID).Error
+		err = db.Model(&User{}).Where("id", alice.ID).UpdateColumn("is_active", true).Error
 		if err != nil {
 			t.Fatal(err)
 		}