Merge pull request #1313 from safing/feature/profile-sync

Add profile merging and prepare for sync

core/base/logs.go

@@ -2,6 +2,7 @@ package base
 
 import (
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -31,7 +32,9 @@ func logCleaner(_ context.Context, _ *modules.Task) error {
 		filepath.Join(dataroot.Root().Path, logFileDir),
 		func(path string, info os.FileInfo, err error) error {
 			if err != nil {
-				log.Warningf("core: failed to access %s while deleting old log files: %s", path, err)
+				if !errors.Is(err, os.ErrNotExist) {
+					log.Warningf("core: failed to access %s while deleting old log files: %s", path, err)
+				}
 				return nil
 			}
 

firewall/master.go

@@ -33,6 +33,7 @@ var defaultDeciders = []deciderFn{
 	checkConnectionType,
 	checkConnectionScope,
 	checkEndpointLists,
+	checkInvalidIP,
 	checkResolverScope,
 	checkConnectivityDomain,
 	checkBypassPrevention,
@@ -371,7 +372,8 @@ func checkConnectionScope(_ context.Context, conn *network.Connection, p *profil
 			return true
 		}
 	case netutils.Undefined, netutils.Invalid:
-		fallthrough
+		// Block Invalid / Undefined IPs _after_ the rules.
+		return false
 	default:
 		conn.Deny("invalid IP", noReasonOptionKey) // Block Outbound / Drop Inbound
 		return true
@@ -380,6 +382,22 @@ func checkConnectionScope(_ context.Context, conn *network.Connection, p *profil
 	return false
 }
 
+func checkInvalidIP(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
+	// Only applies to IP connections.
+	if conn.Type != network.IPConnection {
+		return false
+	}
+
+	// Block Invalid / Undefined IPs.
+	switch conn.Entity.IPScope { //nolint:exhaustive // Only looking for specific values.
+	case netutils.Undefined, netutils.Invalid:
+		conn.Deny("invalid IP", noReasonOptionKey) // Block Outbound / Drop Inbound
+		return true
+	}
+
+	return false
+}
+
 func checkBypassPrevention(ctx context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
 	if p.PreventBypassing() {
 		// check for bypass protection

firewall/tunnel.go

@@ -160,6 +160,8 @@ func DeriveTunnelOptions(lp *profile.LayeredProfile, proc *process.Process, dest
 	}
 	if !connEncrypted {
 		tunnelOpts.Destination.Regard = tunnelOpts.Destination.Regard.Add(navigator.StateTrusted)
+		// TODO: Add this when all Hubs are on v0.6.21+
+		// tunnelOpts.Destination.Regard = tunnelOpts.Destination.Regard.Add(navigator.StateAllowUnencrypted)
 	}
 
 	// Add required verified owners if community nodes should not be used.

go.mod

@@ -20,9 +20,9 @@ require (
 	github.com/mitchellh/go-server-timing v1.0.1
 	github.com/oschwald/maxminddb-golang v1.12.0
 	github.com/safing/jess v0.3.1
-	github.com/safing/portbase v0.17.4
+	github.com/safing/portbase v0.17.5
 	github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec
-	github.com/safing/spn v0.6.19
+	github.com/safing/spn v0.6.21
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/spf13/cobra v1.7.0
 	github.com/spkg/zipfs v0.7.1

go.sum

@@ -240,12 +240,12 @@ github.com/safing/jess v0.3.1 h1:cMZVhi2whW/YdD98MPLeLIWJndQ7o2QVt2HefQ/ByFA=
 github.com/safing/jess v0.3.1/go.mod h1:aj73Eot1zm2ETkJuw9hJlIO8bRom52uBbsCHemvlZmA=
 github.com/safing/portbase v0.15.2/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y=
 github.com/safing/portbase v0.16.2/go.mod h1:mzNCWqPbO7vIYbbK5PElGbudwd2vx4YPNawymL8Aro8=
-github.com/safing/portbase v0.17.4 h1:4RhItvFujwdfLQVfwvB+VYER33AT//Ywv317Vj01TEQ=
-github.com/safing/portbase v0.17.4/go.mod h1:suLPSjOTqA7iDLozis5OI7PSw+wqJNT8SLvdBhRPlqI=
+github.com/safing/portbase v0.17.5 h1:0gq0tgPLbKlK+xq7WM+Kcutu5HgYIglxBE3QqN5tIAA=
+github.com/safing/portbase v0.17.5/go.mod h1:suLPSjOTqA7iDLozis5OI7PSw+wqJNT8SLvdBhRPlqI=
 github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec h1:oSJY1seobofPwpMoJRkCgXnTwfiQWNfGMCPDfqgAEfg=
 github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE=
-github.com/safing/spn v0.6.19 h1:z4i8hb5FGKjmgSzA4MzJ8mOc0hYp11zgXzujrHwwV5k=
-github.com/safing/spn v0.6.19/go.mod h1:LRWLManSXHTViiDqU2qNy3w07auMuadOnVW8wAB/Cgw=
+github.com/safing/spn v0.6.21 h1:7LhaEbQ7xrPMETerydpbEAVmLmp+etGJWKnW5b6iI0g=
+github.com/safing/spn v0.6.21/go.mod h1:MgWfUDkYqi46A+EcxayLD0tc519KBiVEQ6mfAjHIx/4=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0=

network/connection.go

@@ -488,6 +488,12 @@ func (conn *Connection) GatherConnectionInfo(pkt packet.Packet) (err error) {
 		// Errors are informational and are logged to the context.
 	}
 
+	// Only get process and profile with first real packet.
+	// TODO: Remove when we got full VM/Docker support.
+	if pkt.InfoOnly() {
+		return nil
+	}
+
 	// Get Process and Profile.
 	if conn.process == nil {
 		conn.process, err = process.GetProcessWithProfile(pkt.Ctx(), conn.PID)

profile/fingerprint.go

@@ -4,6 +4,11 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"golang.org/x/exp/slices"
+
+	"github.com/safing/jess/lhash"
+	"github.com/safing/portbase/container"
 )
 
 // # Matching and Scores
@@ -57,6 +62,12 @@ type (
 		Key       string // Key must always fully match.
 		Operation string
 		Value     string
+
+		// MergedFrom holds the ID of the profile from which this fingerprint was
+		// merged from. The merged profile should create a new profile ID derived
+		// from the new fingerprints and add all fingerprints with this field set
+		// to the originating profile ID
+		MergedFrom string
 	}
 
 	// Tag represents a simple key/value kind of tag used in process metadata
@@ -347,3 +358,78 @@ func checkMatchStrength(value int) int {
 	}
 	return value
 }
+
+const (
+	deriveFPKeyIDForItemStart = iota + 1
+	deriveFPKeyIDForType
+	deriveFPKeyIDForKey
+	deriveFPKeyIDForOperation
+	deriveFPKeyIDForValue
+)
+
+func deriveProfileID(fps []Fingerprint) string {
+	// Sort the fingerprints.
+	sortAndCompactFingerprints(fps)
+
+	// Compile data for hashing.
+	c := container.New(nil)
+	c.AppendInt(len(fps))
+	for _, fp := range fps {
+		c.AppendNumber(deriveFPKeyIDForItemStart)
+		if fp.Type != "" {
+			c.AppendNumber(deriveFPKeyIDForType)
+			c.AppendAsBlock([]byte(fp.Type))
+		}
+		if fp.Key != "" {
+			c.AppendNumber(deriveFPKeyIDForKey)
+			c.AppendAsBlock([]byte(fp.Key))
+		}
+		if fp.Operation != "" {
+			c.AppendNumber(deriveFPKeyIDForOperation)
+			c.AppendAsBlock([]byte(fp.Operation))
+		}
+		if fp.Value != "" {
+			c.AppendNumber(deriveFPKeyIDForValue)
+			c.AppendAsBlock([]byte(fp.Value))
+		}
+	}
+
+	// Hash and return.
+	h := lhash.Digest(lhash.SHA3_256, c.CompileData())
+	return h.Base58()
+}
+
+func sortAndCompactFingerprints(fps []Fingerprint) []Fingerprint {
+	// Sort.
+	slices.SortFunc[[]Fingerprint, Fingerprint](fps, func(a, b Fingerprint) int {
+		switch {
+		case a.Type != b.Type:
+			return strings.Compare(a.Type, b.Type)
+		case a.Key != b.Key:
+			return strings.Compare(a.Key, b.Key)
+		case a.Operation != b.Operation:
+			return strings.Compare(a.Operation, b.Operation)
+		case a.Value != b.Value:
+			return strings.Compare(a.Value, b.Value)
+		case a.MergedFrom != b.MergedFrom:
+			return strings.Compare(a.MergedFrom, b.MergedFrom)
+		default:
+			return 0
+		}
+	})
+
+	// De-duplicate.
+	// Important: Even if the fingerprint is the same, but MergedFrom is
+	// different, we need to keep the separate fingerprint, so that new installs
+	// will cleanly update to the synced state: Auto-generated profiles need to
+	// be automatically replaced by the merged version.
+	fps = slices.CompactFunc[[]Fingerprint, Fingerprint](fps, func(a, b Fingerprint) bool {
+		return a.Type == b.Type &&
+			a.Key == b.Key &&
+			a.Operation == b.Operation &&
+			a.Value == b.Value &&
+			a.MergedFrom == b.MergedFrom
+	})
+
+	return fps
+}

profile/fingerprint_test.go

@@ -0,0 +1,53 @@
+package profile
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDeriveProfileID(t *testing.T) {
+	t.Parallel()
+
+	fps := []Fingerprint{
+		{
+			Type:      FingerprintTypePathID,
+			Operation: FingerprintOperationEqualsID,
+			Value:     "/sbin/init",
+		},
+		{
+			Type:      FingerprintTypePathID,
+			Operation: FingerprintOperationPrefixID,
+			Value:     "/",
+		},
+		{
+			Type:      FingerprintTypeEnvID,
+			Key:       "PORTMASTER_PROFILE",
+			Operation: FingerprintOperationEqualsID,
+			Value:     "TEST-1",
+		},
+		{
+			Type:      FingerprintTypeTagID,
+			Key:       "tag-key-1",
+			Operation: FingerprintOperationEqualsID,
+			Value:     "tag-key-2",
+		},
+	}
+
+	// Create rand source for shuffling.
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec
+
+	// Test 100 times.
+	for i := 0; i < 100; i++ {
+		// Shuffle fingerprints.
+		rnd.Shuffle(len(fps), func(i, j int) {
+			fps[i], fps[j] = fps[j], fps[i]
+		})
+
+		// Check if fingerprint matches.
+		id := deriveProfileID(fps)
+		assert.Equal(t, "PTSRP7rdCnmvdjRoPMTrtjj7qk7PxR1a9YdBWUGwnZXJh2", id)
+	}
+}

profile/icon.go

@@ -0,0 +1,57 @@
+package profile
+
+import (
+	"strings"
+
+	"golang.org/x/exp/slices"
+)
+
+// Icon describes an icon.
+type Icon struct {
+	Type  IconType
+	Value string
+}
+
+// IconType describes the type of an Icon.
+type IconType string
+
+// Supported icon types.
+const (
+	IconTypeFile     IconType = "path"
+	IconTypeDatabase IconType = "database"
+)
+
+func (t IconType) sortOrder() int {
+	switch t {
+	case IconTypeDatabase:
+		return 1
+	case IconTypeFile:
+		return 2
+	default:
+		return 100
+	}
+}
+
+func sortAndCompactIcons(icons []Icon) []Icon {
+	// Sort.
+	slices.SortFunc[[]Icon, Icon](icons, func(a, b Icon) int {
+		aOrder := a.Type.sortOrder()
+		bOrder := b.Type.sortOrder()
+
+		switch {
+		case aOrder != bOrder:
+			return aOrder - bOrder
+		case a.Value != b.Value:
+			return strings.Compare(a.Value, b.Value)
+		default:
+			return 0
+		}
+	})
+
+	// De-duplicate.
+	icons = slices.CompactFunc[[]Icon, Icon](icons, func(a, b Icon) bool {
+		return a.Type == b.Type && a.Value == b.Value
+	})
+
+	return icons
+}