fix missed data in coins, populate them manually / from other apis (#36)

application.yaml

@@ -37,6 +37,7 @@ cmd/heimdall-identity-io:
 
 cmd/heimdall-asset-data-syncer:
   version: local
+  defaultEndpointTimeout: 60s
   httpServer:
     port: 8888
     certPath: cmd/heimdall-identity-io/.testdata/localhost.crt

cmd/heimdall-asset-data-syncer/contract.go

@@ -8,6 +8,7 @@ type (
 	service struct {
 		coinSyncer coins.Sync
 	}
+	noAuth struct{}
 )
 
 const (

cmd/heimdall-asset-data-syncer/heimdall_asset_data_syncer.go

@@ -30,7 +30,7 @@ func main() {
 
 	appcfg.MustLoadFromKey(applicationYamlKey, &cfg)
 	log.Info(fmt.Sprintf("starting version `%v`...", cfg.Version))
-	server.New(&service{}, applicationYamlKey, "").ListenAndServe(ctx, cancel, nil)
+	server.New(&service{}, applicationYamlKey, "").ListenAndServe(ctx, cancel, new(noAuth))
 }
 
 func (s *service) RegisterRoutes(router *server.Router) {
@@ -53,3 +53,7 @@ func (s *service) CheckHealth(ctx context.Context) error {
 
 	return errors.Wrapf(s.coinSyncer.HealthCheck(ctx), "coins sync check failed")
 }
+
+func (n *noAuth) VerifyToken(ctx context.Context, token string) (server.Token, error) {
+	return nil, errors.Errorf("auth disabled")
+}

coins/coins.go

@@ -36,6 +36,7 @@ func New(ctx context.Context) Coins {
 	if c.needToSyncAllCoins(ctx) {
 		log.Panic(errors.Wrapf(c.syncAllCoins(ctx), "failed to sync all coin gecko coins on startup"))
 	}
+
 	return &c
 }
 
@@ -117,19 +118,31 @@ func (c *coinsRepository) buildInsertBatchForCoins(now *time.Time, coinsList []*
 	placeholders := make([]string, 0, len(coinsList))
 	idx := 2
 	for _, coinItem := range coinsList {
-		decimals := 0
-		params = append(params, c.syncFrequency(coinItem.ID), decimals, generateInternalID(coinItem, nil), coinItem.MappedNetwork(), coinItem.Name, coinItem.Symbol, coinItem.SymbolGroup(), coinItem.ContractAddress, coinItem.ID, coinItem.PriceUSD, coinItem.IconUrl)
+		params = append(params, c.syncFrequency(coinItem.ID), coinItem.Decimals, generateInternalID(coinItem, nil), coinItem.MappedNetwork(), coinItem.Name, coinItem.Symbol, coinItem.SymbolGroup(), coinItem.ContractAddress, coinItem.ID, coinItem.PriceUSD, coinItem.IconUrl)
 		placeholders = append(placeholders, fmt.Sprintf("($1,$1,$1, $%[1]v::INTERVAL, $%[2]v, COALESCE((SELECT MAX(version) FROM coins),0), $%[3]v,$%[4]v, $%[5]v, $%[6]v, $%[7]v, $%[8]v, $%[9]v, $%[10]v, $%[11]v)", idx, idx+1, idx+2, idx+3, idx+4, idx+5, idx+6, idx+7, idx+8, idx+9, idx+10))
 		idx += 11
 	}
 	return strings.Join(placeholders, ", "), params
 }
 
-func generateInternalID(c *coingecko.Coin, mapping map[string]string) string {
-	if c.ID == "" && len(mapping) > 0 {
-		return mapping[c.Network+":"+c.ContractAddress]
+func generateInternalID(coin *coingecko.Coin, mapping map[string]string) string {
+	if coin.ID == "" && len(mapping) > 0 {
+		coin.ID = mapping[coin.Network+":"+coin.ContractAddress]
+	}
+	if coin.Network == "" && len(mapping) > 0 {
+		nw, hasNetwork := mapping[coin.ID]
+		if hasNetwork {
+			coin.Network = nw
+			spl := strings.Split(nw, ":")
+			if len(spl) == 2 {
+				coin.Network = spl[0]
+				if coin.ContractAddress == "" {
+					coin.ContractAddress = spl[1]
+				}
+			}
+		}
 	}
-	hash := md5.Sum([]byte(c.Network + c.ContractAddress + c.ID))
+	hash := md5.Sum([]byte(coin.Network + coin.ContractAddress + coin.ID))
 	id, _ := uuid.FromBytes(hash[:])
 	return id.String()
 }

coins/internal/coingecko/coingecko.go

@@ -53,35 +53,66 @@ func (c *client) ListCoins(ctx context.Context) ([]*Coin, error) {
 	}
 	res := make(map[string][]*Coin)
 	coinsToSyncMarketData := []string{}
+	tokenSyncDecimals := map[string][]string{}
+	platforms, err := c.getAllPlatforms(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get supported platforms")
+	}
+	nativeCoinsToNetwork := make(map[string][]string)
+	for pl, network := range platformToNetworkMapping {
+		if plat, has := platforms[pl]; !has {
+			if _, hasManualMapping := platformToCoinMapping[pl]; hasManualMapping {
+				nativeCoinsToNetwork[platformToCoinMapping[pl]] = append(nativeCoinsToNetwork[platformToCoinMapping[pl]], network)
+			}
+		} else {
+			nativeCoinsToNetwork[plat.NativeCoinId] = append(nativeCoinsToNetwork[plat.NativeCoinId], network)
+		}
+
+	}
 	for _, coin := range *coinList {
 		if len(coin.Platforms) == 0 {
-			res[coin.ID] = append(res[coin.ID], &Coin{
-				ID:              coin.ID,
-				Symbol:          coin.Symbol,
-				Name:            coin.Name,
-				Network:         "",
-				ContractAddress: "",
-			})
+			networks := nativeCoinsToNetwork[coin.ID]
+			if len(networks) == 0 {
+				continue
+			}
+			for _, n := range networks {
+				res[coin.ID] = append(res[coin.ID], &Coin{
+					ID:              coin.ID,
+					Symbol:          coin.Symbol,
+					Name:            coin.Name,
+					Network:         n,
+					ContractAddress: "",
+					Decimals:        platformToDecimalsMapping[networkToPlatformMapping[n]],
+				})
+			}
 			coinsToSyncMarketData = append(coinsToSyncMarketData, coin.ID)
 			continue
 		}
 		coinsToSyncMarketData = append(coinsToSyncMarketData, coin.ID)
+		platformIdx := 0
 		for platform, tokenAddr := range coin.Platforms {
 			if _, has := platformToNetworkMapping[platform]; !has {
 				continue
 			}
+			if platformIdx == 0 && tokenAddr != "" && !strings.Contains(tokenAddr, "/") {
+				tokenSyncDecimals[platformToNetworkMapping[platform]] = append(tokenSyncDecimals[platformToNetworkMapping[platform]], tokenAddr)
+			}
 			network := platformToNetworkMapping[platform]
 			res[coin.ID] = append(res[coin.ID], &Coin{
 				ID:              coin.ID,
 				Symbol:          coin.Symbol,
 				Name:            coin.Name,
 				Network:         network,
 				ContractAddress: tokenAddr,
+				Decimals:        platformToDecimalsMapping[platform],
 			})
+			platformIdx += 1
 		}
 	}
 	log.Debug(fmt.Sprintf("Initially got %v coins/tokens from coingecko, enhancing with market data...", len(res)))
-
+	if res, err = c.enhanceWithTokenData(ctx, res, tokenSyncDecimals); err != nil {
+		return nil, errors.Wrapf(err, "failed to get tokens data on initial sync")
+	}
 	return c.enhanceWithMarketData(ctx, res, coinsToSyncMarketData)
 }
 
@@ -102,7 +133,7 @@ func (c *client) enhanceWithMarketData(ctx context.Context, coins map[string][]*
 		log.Debug(fmt.Sprintf("Fetching market data for coins %v/%v...", i+1, len(batches)))
 		coinsWithPrice, err := c.GetCoins(ctx, batch)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to fecth market data for coins %+v", batch)
+			return nil, errors.Wrapf(err, "failed to fetchh market data for coins %+v", batch)
 		}
 		for _, coin := range coinsWithPrice {
 			updCoins := coins[coin.ID]
@@ -120,6 +151,43 @@ func (c *client) enhanceWithMarketData(ctx context.Context, coins map[string][]*
 	return res, nil
 }
 
+func (c *client) enhanceWithTokenData(ctx context.Context, coins map[string][]*Coin, tokenAddrsByNetwork map[string][]string) (map[string][]*Coin, error) {
+	networkIdx := 0
+	if len(tokenAddrsByNetwork) == 0 {
+		return map[string][]*Coin{}, nil
+	}
+	for network, tokenAddrs := range tokenAddrsByNetwork {
+		var batches [][]string
+		if len(tokenAddrs) > MaxTokenAddrsGetTokenData {
+			for len(tokenAddrs) > MaxTokenAddrsGetTokenData {
+				batches = append(batches, tokenAddrs[:MaxTokenAddrsGetTokenData])
+				tokenAddrs = tokenAddrs[MaxTokenAddrsGetTokenData+1:]
+			}
+			if len(tokenAddrs) > 0 {
+				batches = append(batches, tokenAddrs)
+			}
+		} else {
+			batches = append(batches, tokenAddrs)
+		}
+		for i, batch := range batches {
+			log.Debug(fmt.Sprintf("Fetching data for tokens on %v (%v/%v) %v/%v...", network, networkIdx, len(tokenAddrsByNetwork), i+1, len(batches)))
+			tokensData, err := c.GetTokens(ctx, network, batch)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to fetch data for tokens %v %+v", batch)
+			}
+			for _, coin := range tokensData {
+				updCoins := coins[coin.ID]
+				for _, resCoin := range updCoins {
+					resCoin.Decimals = coin.Decimals
+				}
+				coins[coin.ID] = updCoins
+			}
+		}
+		networkIdx += 1
+	}
+	return coins, nil
+}
+
 func (c *client) GetCoins(ctx context.Context, coinIDs []string) ([]*Coin, error) {
 	coinsData, _, err := makeAPICall[[]coin](ctx, c, "/api/v3/coins/markets",
 		map[string]any{"ids": strings.Join(coinIDs, ","), "vs_currency": "USD", "per_page": maxCoinsPerPage})
@@ -133,6 +201,7 @@ func (c *client) GetCoins(ctx context.Context, coinIDs []string) ([]*Coin, error
 			Name:     coinData.Name,
 			PriceUSD: float64(coinData.CurrentPrice),
 			IconUrl:  coinData.Image,
+			Symbol:   coinData.Symbol,
 		})
 	}
 	return res, nil
@@ -217,7 +286,22 @@ func convertTokenData(network string, tok tokenData) *Coin {
 	}
 }
 
-func backoff(_ *req.Response, attempt int) stdlibtime.Duration {
+func (c *client) getAllPlatforms(ctx context.Context) (map[string]*platform, error) {
+	platforms, _, err := makeAPICall[[]*platform](ctx, c, "/api/v3/asset_platforms", make(map[string]any))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to fetch supported platforms")
+	}
+	res := map[string]*platform{}
+	for _, pl := range *platforms {
+		res[pl.Id] = pl
+	}
+	return res, nil
+}
+
+func backoff(resp *req.Response, attempt int) stdlibtime.Duration {
+	if resp != nil && resp.StatusCode == http.StatusTooManyRequests {
+		return 61 * stdlibtime.Second
+	}
 	switch {
 	case attempt <= 1:
 		return 100 * stdlibtime.Millisecond //nolint:gomnd // .
@@ -286,7 +370,6 @@ func (c *Coin) SymbolGroup() string {
 	return c.ID
 }
 func MapNetwork(network string) (string, error) {
-
 	if coingeckoNetwork, hasNetwork := networksMapping[strings.ToLower(network)]; !hasNetwork || coingeckoNetwork == "" {
 		return "", ErrInvalidNetwork
 	} else {

coins/internal/coingecko/contract.go

@@ -65,10 +65,8 @@ var (
 		"fantomtestnet":     "ftm",
 		"icp (aka dfinity)": "icp",
 		"kusama":            "kusama",
-		"westend":           "polkadot",
 		"optimism":          "optimism",
 		"optimismsepolia":   "optimism",
-		"polkadot":          "polkadot",
 		"seipacific1":       "sei-network",
 		"seiatlantic2":      "sei-network",
 		"solana":            "solana",
@@ -90,6 +88,8 @@ var (
 		//"stellartestnet":    "stellar",
 		//"stellar":   	       "stellar",
 		//"kaspa":             "kaspa",
+		//"polkadot":          "polkadot",
+		//"westend":           "polkadot",
 	}
 	platformToNetworkMapping = map[string]string{
 		"ethereum":            "eth",
@@ -107,20 +107,50 @@ var (
 		"the-open-network":    "ton",
 		"tron":                "tron",
 		"cardano":             "cardano",
-		"polkadot":            "polkadot",
 		"sei-network":         "sei-network",
 		"internet-computer":   "icp",
 		// Those networks below are not presented on /api/v3/onchain/networks on coingecko
 		// We cannot req tokens on them, it responds 404
 		//"tezos": "tezos",
 		//"kasplex":             "kaspa",
+		//"polkadot":            "polkadot",
 	}
 	networkToPlatformMapping map[string]string
+
+	platformToDecimalsMapping = map[string]int{
+		"ethereum":            18,
+		"base":                18,
+		"bitcoin":             8,
+		"binance-smart-chain": 18,
+		"polygon-pos":         18,
+		"avalanche":           18,
+		"fantom":              18,
+		"arbitrum-one":        18,
+		"optimistic-ethereum": 18,
+		"solana":              9,
+		"kava":                6,
+		"kusama":              12,
+		"the-open-network":    9,
+		"tron":                18,
+		"cardano":             18,
+		"sei-network":         18,
+		"internet-computer":   18,
+		// Those networks below are not presented on /api/v3/onchain/networks on coingecko
+		// We cannot req tokens on them, it responds 404
+		//"tezos":               6,
+		//"kasplex":             8,
+		//"polkadot":            16,
+	}
+
+	platformToCoinMapping = map[string]string{
+		"bitcoin": "bitcoin",
+	}
 )
 
 type (
 	client struct {
-		cfg *config
+		cfg                  *config
+		nativeCoinsToNetwork map[string][]string
 	}
 	config struct {
 		CoinGecko struct {
@@ -169,6 +199,10 @@ type (
 			} `json:"attributes"`
 		} `json:"data"`
 	}
+	platform struct {
+		Id           string `json:"id"`
+		NativeCoinId string `json:"native_coin_id"`
+	}
 )
 
 const (