EdgexAgent/device-ble-go/vendor/github.com/openziti/sdk-golang/edge-apis/pool.go

/*
	Copyright 2019 NetFoundry Inc.

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	https://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
*/

package edge_apis

import (
	"github.com/go-openapi/runtime"
	"github.com/michaelquigley/pfxlog"
	cmap "github.com/orcaman/concurrent-map/v2"
	errors "github.com/pkg/errors"
	"math/rand/v2"
	"net"
	"net/url"
	"sync/atomic"
	"time"
)

type ApiClientTransport struct {
	runtime.ClientTransport
	ApiUrl *url.URL
}

// ClientTransportPool abstracts the concept of multiple `runtime.ClientTransport` (openapi interface) representing one
// target OpenZiti network. In situations where controllers are running in HA mode (multiple controllers) this
// interface can attempt to try different controller during outages or partitioning.
type ClientTransportPool interface {
	runtime.ClientTransport

	Add(apiUrl *url.URL, transport runtime.ClientTransport)
	Remove(apiUrl *url.URL)

	GetActiveTransport() *ApiClientTransport
	SetActiveTransport(*ApiClientTransport)
	GetApiUrls() []*url.URL
	IterateTransportsRandomly() chan<- *ApiClientTransport

	TryTransportsForOp(operation *runtime.ClientOperation) (any, error)
	TryTransportForF(cb func(*ApiClientTransport) (any, error)) (any, error)
}

var _ runtime.ClientTransport = (ClientTransportPool)(nil)
var _ ClientTransportPool = (*ClientTransportPoolRandom)(nil)

// ClientTransportPoolRandom selects a client transport (controller) at random until it is unreachable. Controllers
// are tried at random until a controller is reached. The newly connected controller is set for use on future requests
// until is too becomes unreachable.
type ClientTransportPoolRandom struct {
	pool    cmap.ConcurrentMap[string, *ApiClientTransport]
	current atomic.Pointer[ApiClientTransport]
}

func (c *ClientTransportPoolRandom) IterateTransportsRandomly() chan<- *ApiClientTransport {
	channel := make(chan *ApiClientTransport, 1)

	go func() {
		var transports []*ApiClientTransport

		for tpl := range c.pool.IterBuffered() {
			transports = append(transports, tpl.Val)
		}

		for len(transports) > 0 {
			var selected *ApiClientTransport
			selected, transports = selectAndRemoveRandom(transports, nil)

			if selected != nil {
				channel <- selected
			}
		}
	}()

	return channel
}

func (c *ClientTransportPoolRandom) GetApiUrls() []*url.URL {
	var result []*url.URL

	for tpl := range c.pool.IterBuffered() {
		result = append(result, tpl.Val.ApiUrl)
	}

	return result
}

func (c *ClientTransportPoolRandom) GetActiveTransport() *ApiClientTransport {
	active := c.current.Load()
	if active == nil {
		active = c.AnyTransport()
		c.SetActiveTransport(active)
	}

	return active
}

func (c *ClientTransportPoolRandom) GetApiClientTransports() []*ApiClientTransport {
	var result []*ApiClientTransport

	for tpl := range c.pool.IterBuffered() {
		result = append(result, tpl.Val)
	}

	return result
}

func NewClientTransportPoolRandom() *ClientTransportPoolRandom {
	return &ClientTransportPoolRandom{
		pool:    cmap.New[*ApiClientTransport](),
		current: atomic.Pointer[ApiClientTransport]{},
	}
}

func (c *ClientTransportPoolRandom) SetActiveTransport(transport *ApiClientTransport) {
	pfxlog.Logger().WithField("key", transport.ApiUrl.String()).Debug("setting active controller")
	c.current.Store(transport)
}

func (c *ClientTransportPoolRandom) Add(apiUrl *url.URL, transport runtime.ClientTransport) {
	c.pool.Set(apiUrl.String(), &ApiClientTransport{
		ClientTransport: transport,
		ApiUrl:          apiUrl,
	})
}

func (c *ClientTransportPoolRandom) Remove(apiUrl *url.URL) {
	c.pool.Remove(apiUrl.String())
}

func (c *ClientTransportPoolRandom) Submit(operation *runtime.ClientOperation) (any, error) {
	return c.TryTransportsForOp(operation)
}

func (c *ClientTransportPoolRandom) TryTransportsForOp(operation *runtime.ClientOperation) (any, error) {
	result, err := c.TryTransportForF(func(transport *ApiClientTransport) (any, error) {
		return transport.Submit(operation)
	})

	return result, err
}

func (c *ClientTransportPoolRandom) IterateRandomTransport() <-chan *ApiClientTransport {
	var transportsToTry []*cmap.Tuple[string, *ApiClientTransport]
	for tpl := range c.pool.IterBuffered() {
		transportsToTry = append(transportsToTry, &tpl)
	}

	ch := make(chan *ApiClientTransport, len(transportsToTry))

	go func() {
		for len(transportsToTry) > 0 {
			var transportTpl *cmap.Tuple[string, *ApiClientTransport]
			transportTpl, transportsToTry = selectAndRemoveRandom(transportsToTry, nil)
			ch <- transportTpl.Val
		}
	}()

	return ch
}

func (c *ClientTransportPoolRandom) TryTransportForF(cb func(*ApiClientTransport) (any, error)) (any, error) {
	//try active first if we have it
	active := c.GetActiveTransport()
	activeKey := ""

	if active != nil {
		activeKey = active.ApiUrl.String()
		result, err := cb(active)

		if err == nil {
			return result, err
		}

		if !errorIndicatesControllerSwap(err) {
			pfxlog.Logger().WithError(err).Debugf("determined that error (%T) does not indicate controller swap, returning error", err)
			return result, err
		}

		pfxlog.Logger().WithError(err).Debugf("encountered error (%T) while submitting request indicating controller swap", err)

		if c.pool.Count() == 1 {
			pfxlog.Logger().Debug("active transport failed, only 1 transport in pool")

			return result, err
		}
	}

	// either no active or active failed, lets start trying them at random
	pfxlog.Logger().Debug("trying random transports from pool")

	ch := c.IterateRandomTransport()

	var lastResult any
	lastErr := errors.New("no transports to try, active transport already failed or was nil") //default err should never be returned
	attempts := 0
	for transport := range ch {
		// skip the already attempted active key
		if activeKey != "" && transport.ApiUrl.String() == activeKey {
			continue
		}

		attempts = attempts + 1
		lastResult, lastErr = cb(transport)

		if lastErr == nil {
			c.SetActiveTransport(transport)
			return lastResult, nil
		}
	}

	return lastResult, lastErr
}

func (c *ClientTransportPoolRandom) AnyTransport() *ApiClientTransport {
	transportBuffer := c.pool.Items()
	var keys []string

	for key := range transportBuffer {
		keys = append(keys, key)
	}

	if len(keys) == 0 {
		return nil
	}
	seed := uint64(time.Now().UnixNano())
	rng := rand.New(rand.NewPCG(seed, seed))
	index := rng.IntN(len(keys))
	return transportBuffer[keys[index]]
}

var _ runtime.ClientTransport = (*ClientTransportPoolRandom)(nil)
var _ ClientTransportPool = (*ClientTransportPoolRandom)(nil)

var opError = &net.OpError{}

func errorIndicatesControllerSwap(err error) bool {
	pfxlog.Logger().WithError(err).Debugf("checking for network errror on type (%T) and its wrapped errors", err)

	if errors.As(err, &opError) {
		pfxlog.Logger().Debug("detected net.OpError")
		return true
	}

	//others? rate limiting? http timeout?

	return false
}

func selectAndRemoveRandom[T any](slice []T, zero T) (selected T, modifiedSlice []T) {
	if len(slice) == 0 {
		return zero, slice
	}
	seed := uint64(time.Now().UnixNano())
	rng := rand.New(rand.NewPCG(seed, seed))
	index := rng.IntN(len(slice))
	selected = slice[index]
	modifiedSlice = append(slice[:index], slice[index+1:]...)
	return selected, modifiedSlice
}