Skip to content

getting connection errors when using worker pool #957

Closed
@n-ozerov

Description

@n-ozerov

Issue description

Getting connection errors when trying to execute sql statements within specific amount of workers

Example code

package main

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"sync"
	"time"
)

var (
	stmt *sql.Stmt
	wg   sync.WaitGroup
)

const WORKERS_COUNT = 140
const WORKERS_MODE = true // false to make Goroutine for each exec

func main() {

	db, err := sql.Open("mysql", "root:pass@/test")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	db.SetMaxOpenConns(140)

	stmt, _ = db.Prepare("INSERT INTO test (zn, idtest) VALUES (?,?)")
	defer stmt.Close()
	dlt, _ := db.Prepare("DELETE FROM test WHERE (idtest > 0)")
	defer dlt.Close()
	dlt.Exec()

	jobs := make(chan int)

	for w := 1; w <= WORKERS_COUNT; w++ {
		go worker(jobs)
	}

	now := time.Now()
	for i := 1; i <= 100000; i++ {
		wg.Add(1)
		if WORKERS_MODE {
			jobs <- i // this does not
		} else {
			go DoSomeJob(i) // this works fine
		}

	}
	wg.Wait()
	fmt.Println(time.Now().Sub(now))
	// time.Sleep(5*time.Second)
}

func DoSomeJob(i int) {

	_, err := stmt.Exec("test", i)

	if err != nil {
		fmt.Println("Error: ", err.Error())
	}

	wg.Done()
}

func worker(jobs chan int) {
	for {
		select {
		case task := <-jobs:
			DoSomeJob(task)
		}
	}
}

MySQL Table:

CREATE TABLE `test` (
  `idtest` int(11) NOT NULL AUTO_INCREMENT,
  `zn` varchar(45) DEFAULT NULL,
  PRIMARY KEY (`idtest`)
) ENGINE=InnoDB AUTO_INCREMENT=100001 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

Error log

getting:
1. dial tcp 127.0.0.1:3306: connectex: Only one usage of each socket address (protocol/network address/port) is normally permitted. -- after about 70k rows when workers_count is less than MaxOpenConns (e.g. 100) 

2. driver: bad connection -- after about 30k row when workers_count is greater than MaxOpenConns

3. works without any errors then workers_count is much greater than MaxOpenConns (e.g. 30000) or less than 80 or using goroutine for each exec

Configuration

*Driver version (or git SHA): d0a5481

Go version: 1.12.4

*Server version: MySQL 8.0.16

*Server OS: Windows 10

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions