init.go

// Package visor pkg/visor/init.go
package visor

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"mime"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	"github.com/ccding/go-stun/stun"
	"github.com/google/uuid"
	"github.com/sirupsen/logrus"
	"github.com/skycoin/dmsg/pkg/direct"
	dmsgdisc "github.com/skycoin/dmsg/pkg/disc"
	"github.com/skycoin/dmsg/pkg/dmsg"
	"github.com/skycoin/dmsg/pkg/dmsgctrl"
	"github.com/skycoin/dmsg/pkg/dmsgcurl"
	"github.com/skycoin/dmsg/pkg/dmsghttp"
	"github.com/skycoin/dmsg/pkg/dmsgpty"

	"github.com/skycoin/skywire-utilities/pkg/cipher"
	"github.com/skycoin/skywire-utilities/pkg/logging"
	"github.com/skycoin/skywire-utilities/pkg/netutil"
	utilenv "github.com/skycoin/skywire-utilities/pkg/skyenv"
	"github.com/skycoin/skywire/internal/vpn"
	"github.com/skycoin/skywire/pkg/app/appdisc"
	"github.com/skycoin/skywire/pkg/app/appevent"
	"github.com/skycoin/skywire/pkg/app/appnet"
	"github.com/skycoin/skywire/pkg/app/appserver"
	"github.com/skycoin/skywire/pkg/app/launcher"
	"github.com/skycoin/skywire/pkg/dmsgc"
	"github.com/skycoin/skywire/pkg/routefinder/rfclient"
	"github.com/skycoin/skywire/pkg/router"
	"github.com/skycoin/skywire/pkg/routing"
	"github.com/skycoin/skywire/pkg/servicedisc"
	"github.com/skycoin/skywire/pkg/skyenv"
	"github.com/skycoin/skywire/pkg/transport"
	"github.com/skycoin/skywire/pkg/transport/network"
	"github.com/skycoin/skywire/pkg/transport/network/addrresolver"
	"github.com/skycoin/skywire/pkg/transport/network/stcp"
	ts "github.com/skycoin/skywire/pkg/transport/setup"
	"github.com/skycoin/skywire/pkg/transport/tpdclient"
	"github.com/skycoin/skywire/pkg/utclient"
	"github.com/skycoin/skywire/pkg/util/osutil"
	"github.com/skycoin/skywire/pkg/visor/dmsgtracker"
	"github.com/skycoin/skywire/pkg/visor/logserver"
	"github.com/skycoin/skywire/pkg/visor/visorconfig"
	vinit "github.com/skycoin/skywire/pkg/visor/visorinit"
)

type visorCtxKey int

const visorKey visorCtxKey = iota

type runtimeErrsCtxKey int

const runtimeErrsKey runtimeErrsCtxKey = iota

const ownerRWX = 0700

// Visor initialization is split into modules, that can be initialized independently
// Modules are declared here as package-level variables, but also need to be registered
// in the modules system: they need init function and dependencies and their name to be set
// To add new piece of functionality to visor, you need to create a new module variable
// and register it properly in registerModules function
var (
	// Event broadcasting system
	ebc vinit.Module
	// Address resolver
	ar vinit.Module
	// App discovery
	disc vinit.Module
	// Stun module
	sc vinit.Module
	// SUDPH module
	sudphC vinit.Module
	// STCPR module
	stcprC vinit.Module
	// STCP module
	stcpC vinit.Module
	// dmsg pty: a remote terminal to the visor working over dmsg protocol
	pty vinit.Module
	// Dmsg module
	dmsgC vinit.Module
	// Transportability checker ensures the visor can accept transports by creating a self-transport or exiting after 3 failed attempts to create one
	tc vinit.Module
	// TPD concurrency checker removes transports from tpd that the visor does not have registered locally
	tpdco vinit.Module
	// Transport manager
	tr vinit.Module
	// Transport setup
	trs vinit.Module
	// Routing system
	rt vinit.Module
	// Application launcher
	launch vinit.Module
	// CLI
	cli vinit.Module
	// hypervisors to control this visor
	hvs vinit.Module
	// Uptime tracker
	ut vinit.Module
	// Public visors: automatically establish connections to public visors
	pvs vinit.Module
	// Public visor: advertise current visor as public
	pv vinit.Module
	// Transport module (this is not a functional module but a grouping of all heavy transport types initializations)
	tm vinit.Module
	// hypervisor module
	hv vinit.Module
	// Dmsg ctrl module
	dmsgCtrl vinit.Module
	// Dmsg http log server module
	dmsgHTTPLogServer vinit.Module
	// System survey module
	systemSurvey vinit.Module
	// Dmsg http module
	dmsgHTTP vinit.Module
	// Dmsg trackers module
	dmsgTrackers vinit.Module
	// Skywire Forwarding conn module
	skyFwd vinit.Module
	// Ping module
	pi vinit.Module
	// visor that groups all modules together
	vis vinit.Module
	// config initialization
//	visorConfig vinit.Module
)

// register all modules: instantiate modules with correct names and dependencies, wrap init
// functions to have access to visor and runtime errors channel
func registerModules(logger *logging.MasterLogger) {
	// utility module maker, to avoid passing logger and wrapping each init function
	// in withVisorCtx
	maker := func(name string, f initFn, deps ...*vinit.Module) vinit.Module {
		return vinit.MakeModule(name, withInitCtx(f), logger, deps...)
	}
	//	visorConfig = maker("visor_config", initVisorConfig)
	dmsgHTTP = maker("dmsg_http", initDmsgHTTP)
	ebc = maker("event_broadcaster", initEventBroadcaster)
	ar = maker("address_resolver", initAddressResolver, &dmsgHTTP)
	disc = maker("discovery", initDiscovery, &dmsgHTTP)
	tr = maker("transport", initTransport, &ar, &ebc, &dmsgHTTP)

	sc = maker("stun_client", initStunClient)
	sudphC = maker("sudph", initSudphClient, &sc, &tr)
	stcprC = maker("stcpr", initStcprClient, &tr)
	stcpC = maker("stcp", initStcpClient, &tr)
	dmsgC = maker("dmsg", initDmsg, &ebc, &dmsgHTTP)
	dmsgCtrl = maker("dmsg_ctrl", initDmsgCtrl, &dmsgC, &tr)
	dmsgHTTPLogServer = maker("dmsghttp_logserver", initDmsgHTTPLogServer, &dmsgC, &tr)
	systemSurvey = maker("system_survey", initSystemSurvey, &dmsgHTTPLogServer)
	dmsgTrackers = maker("dmsg_trackers", initDmsgTrackers, &dmsgC)

	pty = maker("dmsg_pty", initDmsgpty, &dmsgC)
	rt = maker("router", initRouter, &tr, &dmsgC, &dmsgHTTP)
	launch = maker("launcher", initLauncher, &ebc, &disc, &dmsgC, &tr, &rt)
	cli = maker("cli", initCLI)
	hvs = maker("hypervisors", initHypervisors, &dmsgC)
	ut = maker("uptime_tracker", initUptimeTracker, &dmsgHTTP)
	pv = maker("public_autoconnect", initPublicAutoconnect, &tr, &disc)
	trs = maker("transport_setup", initTransportSetup, &dmsgC, &tr)
	tm = vinit.MakeModule("transports", vinit.DoNothing, logger, &sc, &sudphC, &dmsgCtrl, &dmsgHTTPLogServer, &dmsgTrackers, &launch)
	pvs = maker("public_visor", initPublicVisor, &tr, &ar, &disc, &stcprC)
	skyFwd = maker("sky_forward_conn", initSkywireForwardConn, &dmsgC, &dmsgCtrl, &tr, &launch)
	pi = maker("ping", initPing, &dmsgC, &tm)
	tc = maker("transportable", initEnsureVisorIsTransportable, &dmsgC, &tm)
	tpdco = maker("tpd_concurrency", initEnsureTPDConcurrency, &dmsgC, &tm)
	vis = vinit.MakeModule("visor", vinit.DoNothing, logger, &ebc, &ar, &disc, &pty,
		&tr, &rt, &launch, &cli, &hvs, &ut, &pv, &pvs, &trs, &stcpC, &stcprC, &skyFwd, &pi, &systemSurvey, &tc, &tpdco)

	hv = maker("hypervisor", initHypervisor, &vis)
}

type initFn func(context.Context, *Visor, *logging.Logger) error

/*
func initVisorConfig(ctx context.Context, v *Visor, log *logging.Logger) error {
	const ebcTimeout = time.Second
	ebc := appevent.NewBroadcaster(log, ebcTimeout)
	v.pushCloseStack("event_broadcaster", ebc.Close)

	v.initLock.Lock()
	v.ebc = ebc
	v.initLock.Unlock()
	return nil
}
*/

func initDmsgHTTP(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	var keys cipher.PubKeys
	servers := v.conf.Dmsg.Servers

	if len(servers) == 0 {
		return nil
	}

	keys = append(keys, v.conf.PK)
	entries := direct.GetAllEntries(keys, servers)
	dClient := direct.NewClient(entries, v.MasterLogger().PackageLogger("dmsg_http:direct_client"))

	dmsgDC, closeDmsgDC, err := direct.StartDmsg(ctx, v.MasterLogger().PackageLogger("dmsg_http:dmsgDC"),
		v.conf.PK, v.conf.SK, dClient, dmsg.DefaultConfig())
	if err != nil {
		return fmt.Errorf("failed to start dmsg: %w", err)
	}

	dmsgHTTP := http.Client{Transport: dmsghttp.MakeHTTPTransport(ctx, dmsgDC)}

	v.pushCloseStack("dmsg_http", func() error {
		closeDmsgDC()
		return nil
	})

	v.initLock.Lock()
	v.dClient = dClient
	v.dmsgHTTP = &dmsgHTTP
	v.dmsgDC = dmsgDC
	v.initLock.Unlock()
	time.Sleep(time.Duration(len(entries)) * time.Second)
	return nil
}

func initEventBroadcaster(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	const ebcTimeout = time.Second
	ebc := appevent.NewBroadcaster(log, ebcTimeout)
	v.pushCloseStack("event_broadcaster", ebc.Close)

	v.initLock.Lock()
	v.ebc = ebc
	v.initLock.Unlock()
	return nil
}

func initAddressResolver(ctx context.Context, v *Visor, log *logging.Logger) error {
	conf := v.conf.Transport

	httpC, err := getHTTPClient(ctx, v, conf.AddressResolver)
	if err != nil {
		return err
	}

	// only needed for dmsghttp
	pIP, err := getPublicIP(v, conf.AddressResolver)
	if err != nil {
		return err
	}

	arClient, err := addrresolver.NewHTTP(conf.AddressResolver, v.conf.PK, v.conf.SK, httpC, pIP, log, v.MasterLogger())
	if err != nil {
		err = fmt.Errorf("failed to create address resolver client: %w", err)
		return err
	}

	v.initLock.Lock()
	v.arClient = arClient
	v.initLock.Unlock()

	doneCh := make(chan struct{}, 1)
	v.pushCloseStack("address_resolver", func() error {
		doneCh <- struct{}{}
		return nil
	})

	return nil
}

func initDiscovery(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	// Prepare app discovery factory.
	factory := appdisc.Factory{
		Log:  v.MasterLogger().PackageLogger("app_discovery"),
		MLog: v.MasterLogger(),
	}

	conf := v.conf.Launcher

	httpC, err := getHTTPClient(ctx, v, conf.ServiceDisc)
	if err != nil {
		return err
	}

	if conf.ServiceDisc != "" {
		factory.PK = v.conf.PK
		factory.SK = v.conf.SK
		factory.ServiceDisc = conf.ServiceDisc
		factory.DisplayNodeIP = conf.DisplayNodeIP
		factory.Client = httpC
		// only needed for dmsghttp
		pIP, err := getPublicIP(v, conf.ServiceDisc)
		if err != nil {
			return err
		}
		factory.ClientPublicIP = pIP
	}

	v.initLock.Lock()
	v.serviceDisc = factory
	v.initLock.Unlock()
	return nil
}

func initStunClient(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all

	sc := network.GetStunDetails(v.conf.StunServers, log)
	v.initLock.Lock()
	v.stunClient = sc
	v.initLock.Unlock()
	v.stunReadyOnce.Do(func() { close(v.stunReady) })
	return nil
}

func initDmsg(ctx context.Context, v *Visor, log *logging.Logger) (err error) { //nolint:all
	if v.conf.Dmsg == nil {
		return fmt.Errorf("cannot initialize dmsg: empty configuration")
	}

	httpC, err := getHTTPClient(ctx, v, v.conf.Dmsg.Discovery)
	if err != nil {
		return err
	}
	dmsgC := dmsgc.New(v.conf.PK, v.conf.SK, v.ebc, v.conf.Dmsg, httpC, v.MasterLogger())
	wg := new(sync.WaitGroup)
	wg.Add(1)
	go func() {
		defer wg.Done()
		dmsgC.Serve(ctx)
	}()

	v.pushCloseStack("dmsg", func() error {
		if err := dmsgC.Close(); err != nil {
			return err
		}
		wg.Wait()
		return nil
	})

	v.initLock.Lock()
	v.dmsgC = dmsgC
	v.initLock.Unlock()
	return nil
}

func initDmsgCtrl(ctx context.Context, v *Visor, _ *logging.Logger) error {
	dmsgC := v.dmsgC
	if dmsgC == nil {
		return nil
	}

	const dmsgTimeout = time.Second * 20
	logger := dmsgC.Logger().WithField("timeout", dmsgTimeout)
	logger.Debug("Connecting to the dmsg network...")
	select {
	case <-time.After(dmsgTimeout):
		logger.Warn("Failed to connect to the dmsg network, will try again later.")
		go func() {
			<-v.dmsgC.Ready()
			logger.Debug("Connected to the dmsg network.")
			v.tpM.InitDmsgClient(ctx, dmsgC)
		}()
	case <-v.dmsgC.Ready():
		logger.Debug("Connected to the dmsg network.")
		v.tpM.InitDmsgClient(ctx, dmsgC)
	}
	// dmsgctrl setup
	cl, err := dmsgC.Listen(visorconfig.DmsgCtrlPort)
	if err != nil {
		return err
	}
	v.pushCloseStack("dmsgctrl", cl.Close)

	dmsgctrl.ServeListener(cl, 0)
	return nil
}

func initDmsgHTTPLogServer(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	dmsgC := v.dmsgC
	if dmsgC == nil {
		return fmt.Errorf("cannot initialize dmsg log server: dmsg not configured")
	}
	logger := v.MasterLogger().PackageLogger("dmsghttp_logserver")

	var printLog bool
	if v.MasterLogger().GetLevel() == logrus.DebugLevel || v.MasterLogger().GetLevel() == logrus.TraceLevel {
		printLog = true
	}

	//whitelist access to the surveys for the hypervisor, dmsggpty whitelist, and for the surveywhitelist of keys which is fetched from the conf service
	var whitelistedPKs []cipher.PubKey
	if v.conf.SurveyWhitelist != nil {
		whitelistedPKs = append(whitelistedPKs, v.conf.SurveyWhitelist...)
	}
	if v.conf.Hypervisors != nil {
		whitelistedPKs = append(whitelistedPKs, v.conf.Hypervisors...)
	}
	if v.conf.Dmsgpty != nil {
		if v.conf.Dmsgpty.Whitelist != nil {
			whitelistedPKs = append(whitelistedPKs, v.conf.Dmsgpty.Whitelist...)
		}
	}

	lsAPI := logserver.New(logger, v.conf.Transport.LogStore.Location, v.conf.LocalPath, v.conf.DmsgHTTPServerPath, whitelistedPKs, &v.survey, printLog)

	lis, err := dmsgC.Listen(visorconfig.DmsgHTTPPort)
	if err != nil {
		return err
	}
	go func() {
		<-ctx.Done()
		if err := lis.Close(); err != nil {
			logger.WithError(err).Error()
		}
	}()

	logger.WithField("dmsg_addr", fmt.Sprintf("dmsg://%v", lis.Addr().String())).
		Debug("Serving...")
	srv := &http.Server{
		ReadHeaderTimeout: 2 * time.Second,
		IdleTimeout:       30 * time.Second,
		Handler:           lsAPI,
	}

	wg := new(sync.WaitGroup)
	wg.Add(1)

	go func() {
		defer wg.Done()
		err = srv.Serve(lis)
		if errors.Is(err, dmsg.ErrEntityClosed) {
			return
		}
		if err != nil {
			logger.WithError(err).Error("Logserver exited with error.")
		}
	}()
	v.pushCloseStack("dmsghttp.logserver", func() error {
		if err := srv.Close(); err != nil {
			return err
		}
		wg.Wait()
		return nil
	})

	return nil
}

func initSystemSurvey(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	go GenerateSurvey(v, log, true)
	return nil
}

func initDmsgTrackers(ctx context.Context, v *Visor, _ *logging.Logger) error { //nolint:all
	dmsgC := v.dmsgC

	dtm := dmsgtracker.NewDmsgTrackerManager(v.MasterLogger(), dmsgC, 0, 0)
	v.pushCloseStack("dmsg_tracker_manager", func() error {
		return dtm.Close()
	})
	v.initLock.Lock()
	v.dtm = dtm
	v.initLock.Unlock()
	v.dtmReadyOnce.Do(func() { close(v.dtmReady) })
	return nil
}

func initSudphClient(ctx context.Context, v *Visor, log *logging.Logger) error {

	var serviceURL dmsgcurl.URL
	_ = serviceURL.Fill(v.conf.Transport.AddressResolver) //nolint:errcheck
	// don't start sudph if we are connection to AR via dmsghttp
	if serviceURL.Scheme == "dmsg" {
		log.Info("SUDPH transport wont be available under dmsghttp")
		return nil
	}
	if v.stunClient != nil {
		switch v.stunClient.NATType {
		case stun.NATSymmetric, stun.NATSymmetricUDPFirewall:
			log.Warnf("SUDPH transport wont be available as visor is under %v", v.stunClient.NATType.String())
		default:
			v.tpM.InitClient(ctx, network.SUDPH, v.conf.Transport.SudphPort)
		}
	}
	return nil
}

func initStcprClient(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	v.tpM.InitClient(ctx, network.STCPR, v.conf.Transport.StcprPort)
	return nil
}

func initStcpClient(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	if v.conf.STCP != nil {
		v.tpM.InitClient(ctx, network.STCP, 0)
	}
	return nil
}

func initTransport(ctx context.Context, v *Visor, log *logging.Logger) error {

	managerLogger := v.MasterLogger().PackageLogger("transport_manager")

	tpdC, err := connectToTpDisc(ctx, v, managerLogger)
	if err != nil {
		err := fmt.Errorf("failed to create transport discovery client: %w", err)
		return err
	}

	var logS transport.LogStore
	if v.conf.Transport.LogStore.Type == visorconfig.MemoryLogStore {
		logS = transport.InMemoryTransportLogStore()
	} else if v.conf.Transport.LogStore.Type == visorconfig.FileLogStore {
		logS, err = transport.FileTransportLogStore(ctx, v.conf.Transport.LogStore.Location, time.Duration(v.conf.Transport.LogStore.RotationInterval), log)
		if err != nil {
			return err
		}
	} else {
		return fmt.Errorf("invalid store type: %v", v.conf.Transport.LogStore.Type)
	}

	pTps, err := v.conf.GetPersistentTransports()
	if err != nil {
		err := fmt.Errorf("failed to get persistent transports: %w", err)
		return err
	}

	tpMConf := transport.ManagerConfig{
		PubKey:                    v.conf.PK,
		SecKey:                    v.conf.SK,
		DiscoveryClient:           tpdC,
		LogStore:                  logS,
		PersistentTransportsCache: pTps,
	}

	// todo: pass down configuration?
	var table stcp.PKTable
	var listenAddr string
	if v.conf.STCP != nil {
		table = stcp.NewTable(v.conf.STCP.PKTable)
		listenAddr = v.conf.STCP.ListeningAddress
	}
	factory := network.ClientFactory{
		PK:         v.conf.PK,
		SK:         v.conf.SK,
		ListenAddr: listenAddr,
		PKTable:    table,
		ARClient:   v.arClient,
		EB:         v.ebc,
		MLogger:    v.MasterLogger(),
	}
	tpM, err := transport.NewManager(managerLogger, v.arClient, v.ebc, &tpMConf, factory)
	if err != nil {
		err := fmt.Errorf("failed to start transport manager: %w", err)
		return err
	}
	ctx, cancel := context.WithCancel(context.Background())
	wg := new(sync.WaitGroup)
	wg.Add(1)

	go func() {
		defer wg.Done()
		tpM.Serve(ctx)
	}()

	v.pushCloseStack("transport.manager", func() error {
		cancel()
		tpM.Close()
		wg.Wait()
		return nil
	})

	v.initLock.Lock()
	v.tpM = tpM
	v.initLock.Unlock()
	return nil
}

func initTransportSetup(ctx context.Context, v *Visor, log *logging.Logger) error {
	ctx, cancel := context.WithCancel(ctx)
	// To remove the block set by NewTransportListener if dmsg is not initialized
	go func() {
		ts, err := ts.NewTransportListener(ctx, v.conf.PK, v.conf.Transport.TransportSetupPKs, v.dmsgC, v.tpM, v.MasterLogger())
		if err != nil {
			log.Warn(err)
			cancel()
		}
		select {
		case <-ctx.Done():
		default:
			go ts.Serve(ctx)
		}
	}()

	// waiting for at least one transport to initialize
	<-v.tpM.Ready()

	v.pushCloseStack("transport_setup.rpc", func() error {
		cancel()
		return nil
	})
	return nil
}

func initSkywireForwardConn(ctx context.Context, v *Visor, log *logging.Logger) error {
	ctx, cancel := context.WithCancel(ctx)
	// waiting for at least one transport to initialize
	<-v.tpM.Ready()
	connApp := appnet.Addr{
		Net:    appnet.TypeSkynet,
		PubKey: v.conf.PK,
		Port:   routing.Port(skyenv.SkyForwardingServerPort),
	}
	l, err := appnet.ListenContext(ctx, connApp)
	if err != nil {
		cancel()
		return err
	}

	v.pushCloseStack("sky_forwarding", func() error {
		cancel()
		if cErr := l.Close(); cErr != nil {
			log.WithError(cErr).Error("Error closing listener.")
		}
		return nil
	})

	go func() {
		for {
			log.Debug("Accepting sky forwarding conn...")
			conn, err := l.Accept()
			if err != nil {
				if !errors.Is(appnet.ErrClosedConn, err) {
					log.WithError(err).Error("Failed to accept conn")
				}
				return
			}
			log.Debug("Accepted sky forwarding conn")

			v.pushCloseStack("sky_forwarding", func() error {
				cancel()
				if cErr := conn.Close(); cErr != nil {
					log.WithError(cErr).Error("Error closing conn.")
				}
				return nil
			})

			log.Debug("Wrapping conn...")
			wrappedConn, err := appnet.WrapConn(conn)
			if err != nil {
				log.WithError(err).Error("Failed to wrap conn")
				return
			}

			rAddr := wrappedConn.RemoteAddr().(appnet.Addr)
			log.Debugf("Accepted sky forwarding conn on %s from %s", wrappedConn.LocalAddr(), rAddr.PubKey)
			go handleServerConn(log, wrappedConn, v)
		}
	}()

	return nil
}

func handleServerConn(log *logging.Logger, remoteConn net.Conn, v *Visor) {
	buf := make([]byte, 32*1024)
	n, err := remoteConn.Read(buf)
	if err != nil {
		log.WithError(err).Error("Failed to read packet")
		return
	}

	var cMsg clientMsg
	err = json.Unmarshal(buf[:n], &cMsg)
	if err != nil {
		log.WithError(err).Error("Failed to marshal json")
		sendError(log, remoteConn, err)
		return
	}
	log.Debugf("Received: %v", cMsg)

	lHost := fmt.Sprintf("localhost:%v", cMsg.Port)
	ok := isPortRegistered(cMsg.Port, v)
	if !ok {
		log.Errorf("Port :%v not registered", cMsg.Port)
		sendError(log, remoteConn, fmt.Errorf("Port :%v not registered", cMsg.Port))
		return
	}

	ok = isPortAvailable(log, cMsg.Port)
	if ok {
		log.Errorf("Failed to dial port %v", cMsg.Port)
		sendError(log, remoteConn, fmt.Errorf("Failed to dial port %v", cMsg.Port))
		return
	}

	log.Debugf("Forwarding %s", lHost)

	// send nil error to indicate to the remote connection that everything is ok
	sendError(log, remoteConn, nil)

	go forward(log, remoteConn, lHost)
}

// forward reads a http.Request from the remote conn of the requesting visor forwards that request
// to the requested local server and forwards the http.Response from the local server to the requesting
// visor via the remote conn
func forward(log *logging.Logger, remoteConn net.Conn, lHost string) {
	for {
		buf := make([]byte, 32*1024)
		n, err := remoteConn.Read(buf)
		if err != nil {
			log.WithError(err).Error("Failed to read packet")
			closeConn(log, remoteConn)
			return
		}
		req, err := http.ReadRequest(bufio.NewReader(bytes.NewBuffer(buf[:n])))
		if err != nil {
			log.WithError(err).Error("Failed to ReadRequest")
			closeConn(log, remoteConn)
			return
		}
		req.RequestURI = ""
		req.URL.Scheme = "http"
		req.URL.Host = lHost
		client := http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			log.WithError(err).Error("Failed to Do req")
			closeConn(log, remoteConn)
			return
		}
		err = resp.Write(remoteConn)
		if err != nil {
			log.WithError(err).Error("Failed to Write")
			closeConn(log, remoteConn)
			return
		}
	}
}

func sendError(log *logging.Logger, remoteConn net.Conn, sendErr error) {
	var sReply serverReply
	if sendErr != nil {
		sErr := sendErr.Error()
		sReply = serverReply{
			Error: &sErr,
		}
	}

	srvReply, err := json.Marshal(sReply)
	if err != nil {
		log.WithError(err).Error("Failed to unmarshal json")
	}

	_, err = remoteConn.Write([]byte(srvReply))
	if err != nil {
		log.WithError(err).Error("Failed write server msg")
	}

	log.Debugf("Server reply sent %s", srvReply)
	// close conn if we send an error
	if sendErr != nil {
		closeConn(log, remoteConn)
	}
}

func closeConn(log *logging.Logger, conn net.Conn) {
	if err := conn.Close(); err != nil {
		log.WithError(err).Errorf("Error closing client %s connection", conn.RemoteAddr())
	}
}

type clientMsg struct {
	Port int `json:"port"`
}

type serverReply struct {
	Error *string `json:"error,omitempty"`
}

func initPing(ctx context.Context, v *Visor, log *logging.Logger) error {
	ctx, cancel := context.WithCancel(ctx)
	// waiting for at least one transport to initialize
	<-v.tpM.Ready()

	connApp := appnet.Addr{
		Net:    appnet.TypeSkynet,
		PubKey: v.conf.PK,
		Port:   routing.Port(skyenv.SkyPingPort),
	}

	l, err := appnet.ListenContext(ctx, connApp)
	if err != nil {
		cancel()
		return err
	}

	v.pushCloseStack("skywire_ping", func() error {
		cancel()
		if cErr := l.Close(); cErr != nil {
			log.WithError(cErr).Error("Error closing listener.")
		}
		return nil
	})

	go func() {
		for {
			log.Debug("Accepting sky ping conn...")
			conn, err := l.Accept()
			if err != nil {
				if !errors.Is(err, appnet.ErrClosedConn) {
					log.WithError(err).Error("Failed to accept ping conn")
				}
				return
			}
			log.Debug("Accepted sky ping conn")
			log.Debug("Wrapping conn...")
			wrappedConn, err := appnet.WrapConn(conn)
			if err != nil {
				log.WithError(err).Error("Failed to wrap conn")
				return
			}

			rAddr := wrappedConn.RemoteAddr().(appnet.Addr)
			log.Debugf("Accepted sky ping conn on %s from %s", wrappedConn.LocalAddr(), rAddr.PubKey)
			go handlePingConn(log, wrappedConn, v)
		}
	}()
	return nil
}

func handlePingConn(log *logging.Logger, remoteConn net.Conn, v *Visor) {
	for {
		buf := make([]byte, (32+v.pingPcktSize)*1024)
		n, err := remoteConn.Read(buf)
		if err != nil {
			if !errors.Is(err, io.EOF) {
				log.WithError(err).Error("Failed to read packet")
			}
			return
		}
		var size PingSizeMsg
		err = json.Unmarshal(buf[:n], &size)
		if err != nil {
			log.WithError(err).Error("Failed to unmarshal json")
			return
		}

		_, err = remoteConn.Write([]byte("ok"))
		if err != nil {
			log.WithError(err).Error("Failed to write message")
			return
		}
		var ping []byte
		for len(ping) != size.Size {
			n, err = remoteConn.Read(buf)
			if err != nil {
				if !errors.Is(err, io.EOF) {
					log.WithError(err).Error("Failed to read packet")
				}
				return
			}
			ping = append(ping, buf[:n]...)
		}
		var msg PingMsg
		err = json.Unmarshal(ping, &msg)
		if err != nil {
			log.WithError(err).Error("Failed to unmarshal json")
			return
		}
		now := time.Now()
		diff := now.Sub(msg.Timestamp)
		v.pingConns[msg.PingPk].latency <- diff

		log.Debugf("Received: %s", buf[:n])
	}
}

// getRouteSetupHooks aka autotransport
func getRouteSetupHooks(ctx context.Context, v *Visor, log *logging.Logger) []router.RouteSetupHook {
	retrier := netutil.NewRetrier(log, time.Second, time.Second*20, 3, 1.3)
	return []router.RouteSetupHook{
		func(rPK cipher.PubKey, tm *transport.Manager) error {
			establishedTransports, _ := v.Transports([]string{string(network.STCPR), string(network.SUDPH), string(network.DMSG)}, []cipher.PubKey{v.conf.PK}, false) //nolint
			for _, transportSum := range establishedTransports {
				if transportSum.Remote.Hex() == rPK.Hex() {
					log.Debugf("Established transport exist. Type: %s", transportSum.Type)
					return nil
				}
			}

			allTransports, err := v.arClient.Transports(ctx)
			if err != nil {
				log.WithError(err).Warn("failed to fetch AR transport")
			}

			dmsgFallback := func() error {
				return retrier.Do(ctx, func() error {
					_, err := tm.SaveTransport(ctx, rPK, network.DMSG, transport.LabelAutomatic)
					if err != nil {
						log.Debugf("Establishing automatic DMSG transport failed.")
					}
					return err
				})
			}
			// check visor's AR transport
			if allTransports == nil && !v.conf.Transport.PublicAutoconnect {
				// skips if there's no AR transports
				log.Warn("empty AR transports")
				return dmsgFallback()
			}
			transports, ok := allTransports[rPK]
			if !ok {
				log.WithField("pk", rPK.String()).Warn("pk not found in the transports")
				// check if automatic transport is available, if it does,
				// continue with route creation
				return dmsgFallback()
			}
			// try to establish direct connection to rPK (single hop) using SUDPH or STCPR
			trySTCPR := false
			trySUDPH := false

			for _, trans := range transports {
				nType := network.Type(trans)
				if nType == network.STCPR {
					trySTCPR = true
					continue
				}

				// Wait until stun client is ready
				<-v.stunReady

				// skip if SUDPH is under symmetric NAT / under UDP firewall.
				if nType == network.SUDPH && (v.stunClient.NATType == stun.NATSymmetric ||
					v.stunClient.NATType == stun.NATSymmetricUDPFirewall) {
					continue
				}
				trySUDPH = true
			}

			// trying to establish direct connection to rPK using STCPR
			if trySTCPR {
				err := retrier.Do(ctx, func() error {
					_, err := tm.SaveTransport(ctx, rPK, network.STCPR, transport.LabelAutomatic)
					return err
				})
				if err == nil {
					return nil
				}
				log.Debugf("Establishing automatic STCPR transport failed.")
			}
			// trying to establish direct connection to rPK using SUDPH
			if trySUDPH {
				err := retrier.Do(ctx, func() error {
					_, err := tm.SaveTransport(ctx, rPK, network.SUDPH, transport.LabelAutomatic)
					return err
				})
				if err == nil {
					return nil
				}
				log.Debugf("Establishing automatic SUDPH transport failed.")
			}

			return dmsgFallback()
		},
	}
}

func initRouter(ctx context.Context, v *Visor, log *logging.Logger) error {
	conf := v.conf.Routing

	httpC, err := getHTTPClient(ctx, v, conf.RouteFinder)
	if err != nil {
		return err
	}

	rfClient := rfclient.NewHTTP(conf.RouteFinder, time.Duration(conf.RouteFinderTimeout), httpC, v.MasterLogger())
	logger := v.MasterLogger().PackageLogger("router")
	rConf := router.Config{
		Logger:           logger,
		MasterLogger:     v.MasterLogger(),
		PubKey:           v.conf.PK,
		SecKey:           v.conf.SK,
		TransportManager: v.tpM,
		RouteFinder:      rfClient,
		RouteGroupDialer: router.NewSetupNodeDialer(),
		SetupNodes:       conf.RouteSetupNodes,
		RulesGCInterval:  0, // TODO
		MinHops:          v.conf.Routing.MinHops,
	}

	routeSetupHooks := getRouteSetupHooks(ctx, v, log)

	r, err := router.New(v.dmsgC, &rConf, routeSetupHooks)
	if err != nil {
		err := fmt.Errorf("failed to create router: %w", err)
		return err
	}

	serveCtx, cancel := context.WithCancel(context.Background())
	if err := r.Serve(serveCtx); err != nil {
		cancel()
		return err
	}

	v.pushCloseStack("router.serve", func() error {
		cancel()
		return r.Close()
	})

	v.initLock.Lock()
	v.rfClient = rfClient
	v.router = r
	v.initLock.Unlock()

	return nil
}

func initLauncher(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	conf := v.conf.Launcher

	// Prepare proc manager.
	procM, err := appserver.NewProcManager(v.MasterLogger(), &v.serviceDisc, v.ebc, conf.ServerAddr, v.conf.LocalPath)
	if err != nil {
		err := fmt.Errorf("failed to start proc_manager: %w", err)
		return err
	}

	v.pushCloseStack("launcher.proc_manager", procM.Close)

	// Prepare launcher.
	launchConf := launcher.AppLauncherConfig{
		VisorPK:       v.conf.PK,
		Apps:          conf.Apps,
		ServerAddr:    conf.ServerAddr,
		BinPath:       conf.BinPath,
		LocalPath:     v.conf.LocalPath,
		DisplayNodeIP: conf.DisplayNodeIP,
	}

	launchLog := v.MasterLogger().PackageLogger("launcher")

	launch, err := launcher.NewLauncher(launchLog, launchConf, v.dmsgC, v.router, procM)
	if err != nil {
		err := fmt.Errorf("failed to start launcher: %w", err)
		return err
	}

	err = launch.AutoStart(launcher.EnvMap{
		visorconfig.VPNClientName: vpnEnvMaker(v.conf, v.dmsgC, v.dmsgDC, v.tpM.STCPRRemoteAddrs()),
		visorconfig.VPNServerName: vpnEnvMaker(v.conf, v.dmsgC, v.dmsgDC, nil),
	})

	if err != nil {
		err := fmt.Errorf("failed to autostart apps: %w", err)
		return err
	}

	v.initLock.Lock()
	v.procM = procM
	v.appL = launch
	v.initLock.Unlock()

	return nil
}

// Make an env maker function for vpn application
func vpnEnvMaker(conf *visorconfig.V1, dmsgC, dmsgDC *dmsg.Client, tpRemoteAddrs []string) launcher.EnvMaker {
	return func() ([]string, error) {
		var envCfg vpn.DirectRoutesEnvConfig

		if conf.Dmsg != nil {
			envCfg.DmsgDiscovery = conf.Dmsg.Discovery

			log := conf.MasterLogger().PackageLogger("vpn_env_maker")
			r := netutil.NewRetrier(log, 1*time.Second, 10*time.Second, 0, 1)
			err := r.Do(context.Background(), func() error {
				for _, ses := range dmsgC.AllSessions() {
					envCfg.DmsgServers = append(envCfg.DmsgServers, ses.RemoteTCPAddr().String())
				}

				if len(envCfg.DmsgServers) == 0 {
					return errors.New("no dmsg servers found")
				}

				if dmsgDC != nil {
					for _, ses := range dmsgDC.AllSessions() {
						envCfg.DmsgServers = append(envCfg.DmsgServers, ses.RemoteTCPAddr().String())
					}
				}
				return nil
			})

			if err != nil {
				return nil, fmt.Errorf("error getting Dmsg servers: %w", err)
			}
		}

		if conf.Transport != nil {
			envCfg.TPDiscovery = conf.Transport.Discovery
			envCfg.AddressResolver = conf.Transport.AddressResolver
		}

		if conf.Routing != nil {
			envCfg.RF = conf.Routing.RouteFinder
		}

		if conf.UptimeTracker != nil {
			envCfg.UptimeTracker = conf.UptimeTracker.Addr
		}

		if conf.STCP != nil && len(conf.STCP.PKTable) != 0 {
			envCfg.STCPTable = conf.STCP.PKTable
		}

		envCfg.TPRemoteIPs = tpRemoteAddrs

		envMap := vpn.AppEnvArgs(envCfg)

		envs := make([]string, 0, len(envMap))
		for k, v := range envMap {
			envs = append(envs, fmt.Sprintf("%s=%s", k, v))
		}

		return envs, nil
	}
}

func initCLI(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	if v.conf.CLIAddr == "" {
		v.log.Debug("'cli_addr' is not configured, skipping.")
		return nil
	}

	cliL, err := net.Listen("tcp", v.conf.CLIAddr)
	if err != nil {
		return err
	}

	v.pushCloseStack("cli.listener", cliL.Close)

	rpcS, err := newRPCServer(v, "CLI")
	if err != nil {
		err := fmt.Errorf("failed to start rpc server for cli: %w", err)
		return err
	}
	go rpcS.Accept(cliL) // We do not use sync.WaitGroup here as it will never return anyway.

	return nil
}

func initHypervisors(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all

	hvErrs := make(map[cipher.PubKey]chan error, len(v.conf.Hypervisors))
	for _, hv := range v.conf.Hypervisors {
		hvErrs[hv] = make(chan error, 1)
	}

	for hvPK, hvErrs := range hvErrs {
		log := v.MasterLogger().PackageLogger("hypervisor_client").WithField("hypervisor_pk", hvPK)

		addr := dmsg.Addr{PK: hvPK, Port: visorconfig.DmsgHypervisorPort}
		rpcS, err := newRPCServer(v, addr.PK.String()[:shortHashLen])
		if err != nil {
			err := fmt.Errorf("failed to start RPC server for hypervisor %s: %w", hvPK, err)
			return err
		}

		ctx, cancel := context.WithCancel(context.Background())
		wg := new(sync.WaitGroup)
		wg.Add(1)

		go func(hvErrs chan error) {
			defer wg.Done()
			//			var autoPeerIP string
			//			if v.autoPeer {
			//				autoPeerIP = v.autoPeerIP
			//			} else {
			//				autoPeerIP = ""
			//			}
			defer delete(v.connectedHypervisors, hvPK)
			v.connectedHypervisors[hvPK] = true
			ServeRPCClient(ctx, log, v.dmsgC, rpcS, addr, hvErrs)
			//			ServeRPCClient(ctx, log, autoPeerIP, v.dmsgC, rpcS, addr, hvErrs)

		}(hvErrs)

		v.pushCloseStack("hypervisor."+hvPK.String()[:shortHashLen], func() error {
			cancel()
			wg.Wait()
			return nil
		})
	}

	return nil
}

func initUptimeTracker(ctx context.Context, v *Visor, log *logging.Logger) error {
	const tickDuration = 5 * time.Minute

	conf := v.conf.UptimeTracker

	if conf == nil {
		v.log.Debug("'uptime_tracker' is not configured, skipping.")
		return nil
	}

	httpC, err := getHTTPClient(ctx, v, conf.Addr)
	if err != nil {
		return err
	}

	pIP, err := getPublicIP(v, conf.Addr)
	if err != nil {
		return err
	}

	ut, err := utclient.NewHTTP(conf.Addr, v.conf.PK, v.conf.SK, httpC, pIP, v.MasterLogger())
	if err != nil {
		v.log.WithError(err).Warn("Failed to connect to uptime tracker.")
		return nil
	}

	ticker := time.NewTicker(tickDuration)

	go func() {
		for range ticker.C {
			c := context.Background()
			if err := ut.UpdateVisorUptime(c, v.conf.Version); err != nil {
				v.isServicesHealthy.unset()
				log.WithError(err).Warn("Failed to update visor uptime.")
			} else {
				v.isServicesHealthy.set()
			}
		}
	}()

	v.pushCloseStack("uptime_tracker", func() error {
		ticker.Stop()
		return nil
	})

	v.initLock.Lock()
	v.uptimeTracker = ut
	v.initLock.Unlock()

	return nil
}

func initEnsureVisorIsTransportable(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	const tickDuration = 5 * time.Minute
	ticker := time.NewTicker(tickDuration)
	go func() {
		time.Sleep(time.Minute)
		tries := 0
		for range ticker.C {
			tpsummary, err := v.AddTransport(v.conf.PK, "dmsg", 0)
			if err != nil {
				tries++
				v.isServicesHealthy.unset()
				log.WithError(err).Warn(fmt.Sprintf("Visor is not transportable! Attempt %v of 3", tries))
				//reduce tick duration on non nil error
				ticker.Reset(time.Minute)
			} else {
				tries = 0
				v.isServicesHealthy.set()
				err = v.RemoveTransport(tpsummary.ID)
				if err != nil {
					log.WithError(err).Warn("Failed to remove self-transport")
				}
				ticker.Reset(tickDuration)
			}
			if tries == 3 {
				log.WithError(err).Error("Visor is not transportable! 3 failed attempts ; exiting now")
				err = v.Shutdown()
				if err != nil {
					log.WithError(err).Fatal("Failed to shut down gracefully")
				}
			}
		}
	}()

	v.pushCloseStack("transportable", func() error {
		ticker.Stop()
		return nil
	})

	return nil
}

func initEnsureTPDConcurrency(ctx context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	const tickDuration = 5 * time.Minute
	ticker := time.NewTicker(tickDuration)
	go func() {
		time.Sleep(time.Minute)
		for range ticker.C {
			entries, err := v.DiscoverTransportsByPK(v.conf.PK)
			if err != nil {
				v.isServicesHealthy.unset()
				log.WithError(err).Warn("Cannot ensure concurrency with TPD")
				//reduce tick duration on non nil error
				ticker.Reset(time.Minute)
			} else {
				var dtpids []uuid.UUID
				var rmtpids []uuid.UUID
				var tpids []uuid.UUID
				for _, e := range entries {
					if e.Edges[0] != e.Edges[1] {
						dtpids = append(dtpids, e.ID)
					}
				}
				transports, err := v.Transports(nil, nil, false)
				for _, t := range transports {
					if t.Local != t.Remote {
						tpids = append(tpids, t.ID)
					}
				}
				for _, t := range dtpids {
					var found bool
					for _, tt := range tpids {
						if tt == t {
							found = true
						}
					}
					if !found {
						rmtpids = append(rmtpids, t)
					}
				}
				if 0 < len(rmtpids) {
					log.WithError(err).Warn(fmt.Sprintf("Found %v transports in transport discovery not registered locally", len(rmtpids)))
					tpdC, err := connectToTpDisc(ctx, v, v.MasterLogger().PackageLogger("tpd_concurrency"))
					if err != nil {
						log.WithError(err).Warn("failed to create transport discovery client")
					} else {
						for _, rm := range rmtpids {
							err = tpdC.DeleteTransport(ctx, rm)
							if err != nil {
								log.WithError(err).Warn(fmt.Sprintf("Failed to remove transport from tpd %v", rm))
							}
						}
					}
				}
				ticker.Reset(tickDuration)
			}
		}
	}()

	v.pushCloseStack("tpd_concurrency", func() error {
		ticker.Stop()
		return nil
	})
	return nil
}

// advertise this visor as public in service discovery
// this service is not considered critical and always returns true
func initPublicVisor(_ context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	if !v.conf.IsPublic {
		// call Stop() method to clean service discovery for the situation that
		// visor was public, then stop (not normal shutdown), then start as non-public
		v.serviceDisc.VisorUpdater(0).Stop()
		return nil
	}
	logger := v.MasterLogger().PackageLogger("public_visor")
	hasPublic, err := netutil.HasPublicIP()
	if err != nil {
		logger.WithError(err).Warn("Failed to check for existing public IP address")
		return nil
	}
	if !hasPublic {
		logger.Warn("No public IP address found, stopping")
		return nil
	}

	stcpr, ok := v.tpM.Stcpr()
	if !ok {
		logger.Warn("No stcpr client found, stopping")
		return nil
	}
	addr, err := stcpr.LocalAddr()
	if err != nil {
		logger.Warn("Failed to get STCPR local addr")
		return nil
	}
	port, err := netutil.ExtractPort(addr)
	if err != nil {
		logger.Warn("Failed to get STCPR port")
		return nil
	}
	visorUpdater := v.serviceDisc.VisorUpdater(port)
	visorUpdater.Start()

	v.log.Debugf("Sent request to register visor as public")
	v.pushCloseStack("public visor updater", func() error {
		visorUpdater.Stop()
		return nil
	})
	return nil
}

func initDmsgpty(ctx context.Context, v *Visor, log *logging.Logger) error {
	conf := v.conf.Dmsgpty

	if conf == nil {
		log.Debug("'dmsgpty' is not configured, skipping.")
		return nil
	}

	// Unlink dmsg socket files (just in case).
	if conf.CLINet == "unix" {
		if runtime.GOOS == "windows" {
			conf.CLIAddr = dmsgpty.ParseWindowsEnv(conf.CLIAddr)
		}

		if err := osutil.UnlinkSocketFiles(v.conf.Dmsgpty.CLIAddr); err != nil {
			log.Error("insufficient permissions")
			return err
		}
	}

	wl := dmsgpty.NewMemoryWhitelist()

	// Initialize the dmsgpty whitelist
	if err := wl.Add(v.conf.Dmsgpty.Whitelist...); err != nil {
		return err
	}

	// Ensure hypervisors are added to the whitelist.
	if err := wl.Add(v.conf.Hypervisors...); err != nil {
		return err
	}
	// add the visor's own public key to the whitelist to allow local pty
	if err := wl.Add(v.conf.PK); err != nil {
		v.log.Errorf("Cannot add itself to the pty whitelist: %s", err)
	}

	dmsgC := v.dmsgC
	if dmsgC == nil {
		err := errors.New("cannot create dmsgpty with nil dmsg client")
		return err
	}

	pty := dmsgpty.NewHost(dmsgC, wl)

	if ptyPort := conf.DmsgPort; ptyPort != 0 {
		serveCtx, cancel := context.WithCancel(context.Background())
		wg := new(sync.WaitGroup)
		wg.Add(1)

		go func() {
			defer wg.Done()
			runtimeErrors := getErrors(ctx)
			if err := pty.ListenAndServe(serveCtx, ptyPort); err != nil {
				runtimeErrors <- fmt.Errorf("listen and serve stopped: %w", err)
			}
		}()

		v.pushCloseStack("router.serve", func() error {
			cancel()
			wg.Wait()
			return nil
		})

	}

	if conf.CLINet != "" {

		if conf.CLINet == "unix" {
			if err := os.MkdirAll(filepath.Dir(conf.CLIAddr), ownerRWX); err != nil {
				err := fmt.Errorf("failed to prepare unix file for dmsgpty cli listener: %w", err)
				return err
			}
		}

		cliL, err := net.Listen(conf.CLINet, conf.CLIAddr)
		if err != nil {
			err := fmt.Errorf("failed to start dmsgpty cli listener: %w", err)
			return err
		}

		serveCtx, cancel := context.WithCancel(context.Background())
		wg := new(sync.WaitGroup)
		wg.Add(1)

		go func() {
			defer wg.Done()
			runtimeErrors := getErrors(ctx)
			if err := pty.ServeCLI(serveCtx, cliL); err != nil {
				runtimeErrors <- fmt.Errorf("serve cli stopped: %w", err)
			}
		}()

		v.pushCloseStack("router.serve", func() error {
			cancel()
			err := cliL.Close()
			wg.Wait()
			return err
		})
	}

	return nil
}

func initPublicAutoconnect(ctx context.Context, v *Visor, log *logging.Logger) error {
	if !v.conf.Transport.PublicAutoconnect {
		return nil
	}
	serviceDisc := v.conf.Launcher.ServiceDisc
	if serviceDisc == "" {
		serviceDisc = utilenv.ServiceDiscAddr
	}

	// todo: refactor updatedisc: split connecting to services in updatedisc and
	// advertising oneself as a service. Currently, config is tailored to
	// advertising oneself and requires things like port that are not used
	// in connecting to services
	conf := servicedisc.Config{
		Type:          servicedisc.ServiceTypeVisor,
		PK:            v.conf.PK,
		SK:            v.conf.SK,
		Port:          uint16(0),
		DiscAddr:      serviceDisc,
		DisplayNodeIP: v.conf.Launcher.DisplayNodeIP,
	}
	// only needed for dmsghttp
	pIP, err := getPublicIP(v, serviceDisc)
	if err != nil {
		return err
	}
	connector := servicedisc.MakeConnector(conf, 3, v.tpM, v.serviceDisc.Client, pIP, log, v.MasterLogger())

	cctx, cancel := context.WithCancel(ctx)
	v.pushCloseStack("public_autoconnect", func() error {
		cancel()
		return err
	})

	go connector.Run(cctx) //nolint:errcheck

	return nil
}

func initHypervisor(_ context.Context, v *Visor, log *logging.Logger) error { //nolint:all
	if v.conf.Hypervisor == nil {
		v.log.Error("hypervisor config = nil")
		return nil
	}
	ctx, cancel := context.WithCancel(context.Background())

	conf := *v.conf.Hypervisor
	conf.PK = v.conf.PK
	conf.SK = v.conf.SK
	conf.DmsgDiscovery = v.conf.Dmsg.Discovery

	// Prepare hypervisor.
	hv, err := NewHypervisor(conf, v, v.dmsgC)
	if err != nil {
		v.log.Fatalln("Failed to start hypervisor:", err)
	}

	hv.serveDmsg(ctx, v.log)

	// Serve HTTP(s).

	// Needed to work with modern browsers when serving from windows, which need the correct mime type for javascript.
	if err := mime.AddExtensionType(".js", "application/javascript"); err != nil {
		log.Fatalln("Unable to register js mime type.")
	}

	v.log.WithField("addr", conf.HTTPAddr).
		WithField("tls", conf.EnableTLS).
		Info("Serving hypervisor...")
	tls := ""
	if conf.EnableTLS {
		tls = "s"
	}
	v.log.Info(fmt.Sprintf("Hypervisor UI: http%s://127.0.0.1%s", tls, conf.HTTPAddr))

	handler := hv.HTTPHandler()
	srv := &http.Server{ //nolint gosec
		Addr:         conf.HTTPAddr,
		Handler:      handler,
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
	}

	go func() {
		if conf.EnableTLS {
			err = srv.ListenAndServeTLS(conf.TLSCertFile, conf.TLSKeyFile)
		} else {
			err = srv.ListenAndServe()
		}

		// don't print error if local server is closed
		if !errors.Is(err, http.ErrServerClosed) {
			v.log.WithError(err).Error("Hypervisor exited with error.")
		}
	}()

	v.pushCloseStack("hypervisor", func() error {
		err := srv.Shutdown(ctx) //nolint
		cancel()
		return err
	})

	return nil
}

func connectToTpDisc(ctx context.Context, v *Visor, log *logging.Logger) (transport.DiscoveryClient, error) {
	const (
		initBO = 1 * time.Second
		maxBO  = 10 * time.Second
		// trying till success
		tries  = 0
		factor = 1
	)

	conf := v.conf.Transport

	httpC, err := getHTTPClient(ctx, v, conf.Discovery)
	if err != nil {
		return nil, err
	}

	// only needed for dmsghttp
	pIP, err := getPublicIP(v, conf.AddressResolver)
	if err != nil {
		return nil, err
	}

	tpdCRetrier := netutil.NewRetrier(log,
		initBO, maxBO, tries, factor)

	var tpdC transport.DiscoveryClient
	retryFunc := func() error {
		var err error
		tpdC, err = tpdclient.NewHTTP(conf.Discovery, v.conf.PK, v.conf.SK, httpC, pIP, v.MasterLogger())
		if err != nil {
			log.WithError(err).Error("Failed to connect to transport discovery, retrying...")
			return err
		}

		return nil
	}

	if err := tpdCRetrier.Do(context.Background(), retryFunc); err != nil {
		return nil, err
	}

	return tpdC, nil
}

// ErrNoVisorInCtx is returned when visor is not set in module initialization context
var ErrNoVisorInCtx = errors.New("visor not set in module initialization context")

// ErrNoErrorsCtx is returned when errors channel is not set in module initialization context
var ErrNoErrorsCtx = errors.New("errors not set in module initialization context")

// withInitCtx wraps init function and returns a hook that can be used in
// the module system
// Passed context should have visor value under visorKey key, this visor will be used
// in the passed function
// Passed context should have errors channel for module runtime errors. It can be accessed
// through a function call
func withInitCtx(f initFn) vinit.Hook {
	return func(ctx context.Context, log *logging.Logger) error {
		val := ctx.Value(visorKey)
		v, ok := val.(*Visor)
		if !ok && v == nil {
			return ErrNoVisorInCtx
		}
		val = ctx.Value(runtimeErrsKey)
		errs, ok := val.(chan error)
		if !ok && errs == nil {
			return ErrNoErrorsCtx
		}
		return f(ctx, v, log)
	}
}

func getErrors(ctx context.Context) chan error {
	val := ctx.Value(runtimeErrsKey)
	errs, ok := val.(chan error)
	if !ok && errs == nil {
		// ok to panic because with check for this value in withInitCtx
		// probably will never be reached, but better than generic NPE just in case
		panic("runtime errors channel is not set in context")
	}
	return errs
}

func getHTTPClient(ctx context.Context, v *Visor, service string) (*http.Client, error) {

	var serviceURL dmsgcurl.URL
	var delegatedServers []cipher.PubKey
	err := serviceURL.Fill(service)

	if serviceURL.Scheme == "dmsg" {
		if err != nil {
			return nil, fmt.Errorf("provided URL is invalid: %w", err)
		}
		// get delegated servers and add them to the client entry
		servers, err := v.dClient.AvailableServers(ctx)
		if err != nil {
			return nil, fmt.Errorf("error getting AvailableServers: %w", err)
		}
		// randomize dmsg servers list
		rand.Shuffle(len(servers), func(i, j int) {
			servers[i], servers[j] = servers[j], servers[i]
		})
		for _, server := range servers {
			delegatedServers = append(delegatedServers, server.Static)
		}

		clientEntry := &dmsgdisc.Entry{
			Client: &dmsgdisc.Client{
				DelegatedServers: delegatedServers,
			},
			Static: serviceURL.Addr.PK,
		}

		err = v.dClient.PostEntry(ctx, clientEntry)
		if err != nil {
			return nil, fmt.Errorf("error saving clientEntry: %w", err)
		}
		return v.dmsgHTTP, nil
	}
	return &http.Client{
		Transport: &http.Transport{
			DisableKeepAlives: true,
			IdleConnTimeout:   time.Second * 5,
		},
	}, nil
}

func getPublicIP(v *Visor, service string) (string, error) {
	var serviceURL dmsgcurl.URL
	var pIP string
	err := serviceURL.Fill(service)
	// only get the IP if the url is of dmsg
	// else just send empty string as ip
	if serviceURL.Scheme != "dmsg" {
		return pIP, nil
	}
	if err != nil {
		return pIP, fmt.Errorf("provided URL is invalid: %w", err)
	}

	pIP, err = GetIP()
	if err != nil {
		<-v.stunReady
		if v.stunClient.PublicIP != nil {
			pIP = v.stunClient.PublicIP.IP()
			return pIP, nil
		}
		err = fmt.Errorf("cannot fetch public ip")
	}
	if err != nil {
		return pIP, err
	}

	return pIP, nil
}

type ipAPI struct {
	PublicIP string `json:"ip_address"`
}

// GetIP used for getting current IP of visor
func GetIP() (string, error) {
	var resp *http.Response
	var err error

	resp, err = http.Get("https://ip.skycoin.com/")
	if err != nil {
		resp, err = http.Get("https://ip.plaintext.ir/")
		if err != nil {
			return "", err
		}
	}
	defer resp.Body.Close() // nolint

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	var ip ipAPI
	err = json.Unmarshal(body, &ip)
	if err != nil {
		return "", err
	}

	return ip.PublicIP, nil
}