Data race when scheduler is started before server is ready to serve

[rleungx]

==================
WARNING: DATA RACE
Read at 0x00c0067af4a0 by goroutine 733:
  github.com/tikv/pd/server/config.(*PersistOptions).getTTLData()
      /home/runner/work/pd/pd/server/config/persist_options.go:674 +0x56
  github.com/tikv/pd/server/config.(*PersistOptions).getTTLUint()
      /home/runner/work/pd/pd/server/config/persist_options.go:636 +0x6b
  github.com/tikv/pd/server/config.(*PersistOptions).getTTLUintOr()
      /home/runner/work/pd/pd/server/config/persist_options.go:645 +0x68
  github.com/tikv/pd/server/config.(*PersistOptions).GetRegionScheduleLimit()
      /home/runner/work/pd/pd/server/config/persist_options.go:313 +0x9e
  github.com/tikv/pd/server/schedulers.(*balanceRegionScheduler).IsScheduleAllowed()
      /home/runner/work/pd/pd/server/schedulers/balance_region.go:129 +0xe7
  github.com/tikv/pd/server/cluster.(*scheduleController).AllowSchedule()
      /home/runner/work/pd/pd/server/cluster/coordinator.go:797 +0xa7
  github.com/tikv/pd/server/cluster.(*coordinator).runScheduler()
      /home/runner/work/pd/pd/server/cluster/coordinator.go:729 +0x3f5

Previous write at 0x00c0067af4a0 by goroutine 589:
  github.com/tikv/pd/server/config.(*PersistOptions).LoadTTLFromEtcd()
      /home/runner/work/pd/pd/server/config/persist_options.go:690 +0x552
  github.com/tikv/pd/server.(*Server).campaignLeader()
      /home/runner/work/pd/pd/server/server.go:1253 +0x1713
  github.com/tikv/pd/server.(*Server).leaderLoop()
      /home/runner/work/pd/pd/server/server.go:1192 +0x6d5

Goroutine 733 (running) created at:
  github.com/tikv/pd/server/cluster.(*coordinator).addScheduler()
      /home/runner/work/pd/pd/server/cluster/coordinator.go:590 +0x296
  github.com/tikv/pd/server/cluster.(*coordinator).run()
      /home/runner/work/pd/pd/server/cluster/coordinator.go:340 +0x1f04
  github.com/tikv/pd/server/cluster.(*RaftCluster).runCoordinator()
      /home/runner/work/pd/pd/server/cluster/cluster.go:336 +0x15c

Goroutine 589 (running) created at:
  github.com/tikv/pd/server.(*Server).startServerLoop()
      /home/runner/work/pd/pd/server/server.go:502 +0x119
  github.com/tikv/pd/server.(*Server).Run()
      /home/runner/work/pd/pd/server/server.go:484 +0x1f2
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x309
  fmt.Fscanf()
      /opt/hostedtoolcache/go/1.16.5/x64/src/fmt/scan.go:143 +0xee
  fmt.Sscanf()
      /opt/hostedtoolcache/go/1.16.5/x64/src/fmt/scan.go:114 +0x191
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0xa6
  github.com/syndtr/goleveldb/leveldb.openDB()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:125 +0x997
  github.com/syndtr/goleveldb/leveldb.Open()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:197 +0x25a
  fmt.Fscanf()
      /opt/hostedtoolcache/go/1.16.5/x64/src/fmt/scan.go:143 +0xee
  fmt.Sscanf()
      /opt/hostedtoolcache/go/1.16.5/x64/src/fmt/scan.go:114 +0x191
  github.com/syndtr/goleveldb/leveldb/storage.fsParseName()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:643 +0xa6
  github.com/syndtr/goleveldb/leveldb/storage.(*fileStorage).List()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/storage/file_storage.go:458 +0x309
  github.com/syndtr/goleveldb/leveldb.(*session).recover.func1()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/session.go:134 +0xfc
  github.com/syndtr/goleveldb/leveldb.(*session).recover()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/session.go:142 +0x1a7f
  github.com/syndtr/goleveldb/leveldb.Open()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:183 +0x109
  github.com/syndtr/goleveldb/leveldb.OpenFile()
      /home/runner/go/pkg/mod/github.com/syndtr/goleveldb@v1.0.1-0.20190318030020-c3a204f8e965/leveldb/db.go:219 +0xbb
  github.com/tikv/pd/server/kv.NewLeveldbKV()
      /home/runner/work/pd/pd/server/kv/levedb_kv.go:32 +0x84
  github.com/tikv/pd/server/core.NewRegionStorage()
      /home/runner/work/pd/pd/server/core/region_storage.go:59 +0x7a
  github.com/tikv/pd/server.(*Server).startServer()
      /home/runner/work/pd/pd/server/server.go:381 +0x11f7
  github.com/tikv/pd/server.(*Server).Run()
      /home/runner/work/pd/pd/server/server.go:480 +0x152
  github.com/tikv/pd/tests.(*TestServer).Run()
      /home/runner/work/pd/pd/tests/cluster.go:101 +0x131
  github.com/tikv/pd/tests.(*TestCluster).RunServer.func1()
      /home/runner/work/pd/pd/tests/cluster.go:429 +0x54
==================

See https://github.com/tikv/pd/pull/3770/checks?check_run_id=2825935654