Merge pull request #514 from OdyseeTeam/feature-arfleet

Feature arfleet

.github/workflows/test-release.yml

@@ -28,7 +28,7 @@ jobs:
         id: release_details
         run: |
           echo "APP_NAME=$(git describe --tags|sed -e 's/\-v.*//')" >> $GITHUB_ENV
-          echo "APP_VERSION=$(git describe --tags|sed 's/api-v\([0-9.]*\)-.*/\1/')" >> $GITHUB_ENV
+          echo "APP_VERSION=$(git describe --tags --match 'api-v*'|sed 's/api-v\([0-9.]*\)/\1/')" >> $GITHUB_ENV
 
       - name: Release details
         run: |
@@ -38,10 +38,10 @@ jobs:
             echo "::endgroup::"
 
       - name: Set up containers
-        run: docker-compose up -d
+        run: docker compose up -d
 
       - name: Set up containers
-        run: docker-compose -f apps/watchman/docker-compose.yml up -d clickhouse geoipupdate
+        run: docker compose -f apps/watchman/docker-compose.yml up -d clickhouse geoipupdate
 
       - name: Check running containers
         run: docker ps -a

app/arweave/arweave.go

@@ -0,0 +1,81 @@
+package arweave
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
+func ReplaceAssetUrls(baseUrl string, structure any, collPath, itemPath string) (any, error) {
+	var origUrls []string
+	urlPaths := map[string][]string{}
+
+	jsonData, err := json.Marshal(structure)
+	if err != nil {
+		return nil, err
+	}
+
+	items := gjson.GetBytes(jsonData, collPath)
+	items.ForEach(func(key, value gjson.Result) bool {
+		urlPath := fmt.Sprintf("%s.%s.%s", collPath, key.String(), itemPath)
+		url := gjson.GetBytes(jsonData, urlPath).String()
+		origUrls = append(origUrls, url)
+		if slice, exists := urlPaths[url]; exists {
+			urlPaths[url] = append(slice, urlPath)
+		} else {
+			urlPaths[url] = []string{urlPath}
+		}
+		return true
+	})
+
+	resolver := NewAssetResolver(baseUrl)
+	subsUrls, err := resolver.ResolveUrls(origUrls)
+	if err != nil {
+		return nil, err
+	}
+
+	for oldURL, newURL := range subsUrls {
+		for _, path := range urlPaths[oldURL] {
+			jsonData, _ = sjson.SetBytes(jsonData, path, newURL)
+		}
+	}
+
+	var d any
+	return d, json.Unmarshal(jsonData, &d)
+}
+
+func ReplaceAssetUrl(baseUrl string, structure any, path string) (any, error) {
+	jsonData, err := json.Marshal(structure)
+	if err != nil {
+		return nil, err
+	}
+
+	origUrl := gjson.GetBytes(jsonData, path).String()
+
+	resolver := NewAssetResolver(baseUrl)
+	subsUrls, err := resolver.ResolveUrls([]string{origUrl})
+
+	if err != nil {
+		return nil, err
+	}
+	if newUrl, ok := subsUrls[origUrl]; ok {
+		jsonData, err = sjson.SetBytes(jsonData, path, newUrl)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var d any
+	return d, json.Unmarshal(jsonData, &d)
+}
+
+func GetClaimUrl(baseUrl, claim_id string) (string, error) {
+	resolver := NewAssetResolver(baseUrl)
+	r, err := resolver.ResolveClaims([]string{claim_id})
+	if err != nil {
+		return "", err
+	}
+	return r[claim_id], nil
+}

app/arweave/arweave_test.go

@@ -0,0 +1,53 @@
+package arweave
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/ybbus/jsonrpc"
+)
+
+func TestReplaceAssetUrls(t *testing.T) {
+	t.Skip("skipping this in automated mode as it requires extra setup on arfleet")
+
+	require := require.New(t)
+	assert := assert.New(t)
+
+	absPath, _ := filepath.Abs("./testdata/claim_search.json")
+	f, err := os.ReadFile(absPath)
+	require.NoError(err)
+	var resp jsonrpc.RPCResponse
+	require.NoError(json.Unmarshal(f, &resp))
+	result, err := ReplaceAssetUrls("http://odycdn.com", resp.Result, "items", "value.thumbnail.url")
+	require.NoError(err)
+
+	out, err := json.MarshalIndent(result, "", "  ")
+	require.NoError(err)
+	re := regexp.MustCompile(`http://odycdn.com/explore/\w{64}\?filename=\w{64}\.webp`)
+	matches := re.FindAllString(string(out), -1)
+	assert.Equal(2, len(matches))
+}
+
+func TestReplaceAssetUrl(t *testing.T) {
+	t.Skip("skipping this in automated mode as it requires extra setup on arfleet")
+
+	require := require.New(t)
+	assert := assert.New(t)
+
+	absPath, _ := filepath.Abs("./testdata/resolve.json")
+	f, err := os.ReadFile(absPath)
+	require.NoError(err)
+	var resp jsonrpc.RPCResponse
+	require.NoError(json.Unmarshal(f, &resp))
+	result, err := ReplaceAssetUrl("http://odycdn.com", resp.Result.(map[string]any)["lbry://@MySillyReactions#d1ae6a9097b44691d318a5bfc6dc1240311c75e2"], "value.thumbnail.url")
+	require.NoError(err)
+
+	out, err := json.MarshalIndent(result, "", "  ")
+	require.NoError(err)
+	assert.Regexp(`http://odycdn.com/explore/\w{64}\?filename=\w{64}\.jpg`, string(out))
+}

app/arweave/resolve.go

@@ -0,0 +1,135 @@
+package arweave
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+const (
+	batchResolverUrl      = "https://migrator.arfleet.zephyrdev.xyz/batch-resolve"
+	batchClaimResolverUrl = "https://migrator.arfleet.zephyrdev.xyz/claims/batch-resolve"
+	resolverTimeout       = 10 * time.Second
+)
+
+type HttpDoer interface {
+	Do(req *http.Request) (res *http.Response, err error)
+}
+
+type AssetResolver struct {
+	baseUrl               string
+	batchResolverUrl      string
+	batchClaimResolverUrl string
+	client                HttpDoer
+}
+
+type BatchResolveUrlResponse map[string]ResolveUrlResponse
+
+type ResolveUrlResponse struct {
+	URL      string `json:"url"`
+	URLHash  string `json:"url_hash"`
+	Arfleet  string `json:"arfleet"`
+	Resolved bool   `json:"resolved"`
+}
+
+type BatchResolveClaimResponse map[string]ResolveClaimResponse
+type ResolveClaimResponse struct {
+	ClaimId  string `json:"claim_id"`
+	Arfleet  string `json:"arfleet"`
+	Resolved bool   `json:"resolved"`
+}
+
+func NewAssetResolver(baseUrl string) *AssetResolver {
+	r := &AssetResolver{
+		baseUrl:               baseUrl,
+		batchResolverUrl:      batchResolverUrl,
+		batchClaimResolverUrl: batchClaimResolverUrl,
+		client: &http.Client{
+			Timeout: resolverTimeout,
+		},
+	}
+	return r
+}
+
+func (c *AssetResolver) ResolveUrls(urls []string) (map[string]string, error) {
+	substitutes := map[string]string{}
+
+	jsonData, err := json.Marshal(map[string][]string{"urls": urls})
+	if err != nil {
+		return nil, err
+	}
+
+	jsonResponse, err := c.makeRequest(http.MethodPost, c.batchResolverUrl, jsonData)
+	if err != nil {
+		return nil, err
+	}
+	var resolvedList BatchResolveUrlResponse
+	err = json.Unmarshal(jsonResponse, &resolvedList)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing json: %w", err)
+	}
+
+	for url, resolved := range resolvedList {
+		if !resolved.Resolved {
+			continue
+		}
+		substitutes[url] = c.baseUrl + resolved.Arfleet
+	}
+	return substitutes, nil
+}
+
+func (c *AssetResolver) ResolveClaims(claim_ids []string) (map[string]string, error) {
+	substitutes := map[string]string{}
+
+	jsonData, err := json.Marshal(map[string][]string{"claim_ids": claim_ids})
+	if err != nil {
+		return nil, err
+	}
+
+	jsonResponse, err := c.makeRequest(http.MethodPost, c.batchClaimResolverUrl, jsonData)
+	if err != nil {
+		return nil, err
+	}
+	var resolvedList BatchResolveClaimResponse
+	err = json.Unmarshal(jsonResponse, &resolvedList)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing json: %w", err)
+	}
+
+	for url, resolved := range resolvedList {
+		if !resolved.Resolved {
+			continue
+		}
+		substitutes[url] = c.baseUrl + resolved.Arfleet
+	}
+	return substitutes, nil
+}
+
+func (c *AssetResolver) makeRequest(method, url string, jsonData []byte) ([]byte, error) {
+	client := c.client
+
+	req, err := http.NewRequest(method, url, bytes.NewBuffer(jsonData))
+	if err != nil {
+		return nil, fmt.Errorf("error creating request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error sending request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading response: %w", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: got %v, want %v", resp.StatusCode, http.StatusOK)
+	}
+	return body, nil
+}