diff --git a/core/base/logs.go b/core/base/logs.go index 2f116aca2..dab2ebac3 100644 --- a/core/base/logs.go +++ b/core/base/logs.go @@ -2,6 +2,7 @@ package base import ( "context" + "errors" "os" "path/filepath" "strings" @@ -31,7 +32,9 @@ func logCleaner(_ context.Context, _ *modules.Task) error { filepath.Join(dataroot.Root().Path, logFileDir), func(path string, info os.FileInfo, err error) error { if err != nil { - log.Warningf("core: failed to access %s while deleting old log files: %s", path, err) + if !errors.Is(err, os.ErrNotExist) { + log.Warningf("core: failed to access %s while deleting old log files: %s", path, err) + } return nil } diff --git a/firewall/master.go b/firewall/master.go index 3277c658e..4183d5610 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -33,6 +33,7 @@ var defaultDeciders = []deciderFn{ checkConnectionType, checkConnectionScope, checkEndpointLists, + checkInvalidIP, checkResolverScope, checkConnectivityDomain, checkBypassPrevention, @@ -371,7 +372,8 @@ func checkConnectionScope(_ context.Context, conn *network.Connection, p *profil return true } case netutils.Undefined, netutils.Invalid: - fallthrough + // Block Invalid / Undefined IPs _after_ the rules. + return false default: conn.Deny("invalid IP", noReasonOptionKey) // Block Outbound / Drop Inbound return true @@ -380,6 +382,22 @@ func checkConnectionScope(_ context.Context, conn *network.Connection, p *profil return false } +func checkInvalidIP(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { + // Only applies to IP connections. + if conn.Type != network.IPConnection { + return false + } + + // Block Invalid / Undefined IPs. + switch conn.Entity.IPScope { //nolint:exhaustive // Only looking for specific values. + case netutils.Undefined, netutils.Invalid: + conn.Deny("invalid IP", noReasonOptionKey) // Block Outbound / Drop Inbound + return true + } + + return false +} + func checkBypassPrevention(ctx context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { if p.PreventBypassing() { // check for bypass protection diff --git a/firewall/tunnel.go b/firewall/tunnel.go index 0f747e187..cadab2eaf 100644 --- a/firewall/tunnel.go +++ b/firewall/tunnel.go @@ -160,6 +160,8 @@ func DeriveTunnelOptions(lp *profile.LayeredProfile, proc *process.Process, dest } if !connEncrypted { tunnelOpts.Destination.Regard = tunnelOpts.Destination.Regard.Add(navigator.StateTrusted) + // TODO: Add this when all Hubs are on v0.6.21+ + // tunnelOpts.Destination.Regard = tunnelOpts.Destination.Regard.Add(navigator.StateAllowUnencrypted) } // Add required verified owners if community nodes should not be used. diff --git a/go.mod b/go.mod index 6135b0263..67de66a6f 100644 --- a/go.mod +++ b/go.mod @@ -20,9 +20,9 @@ require ( github.com/mitchellh/go-server-timing v1.0.1 github.com/oschwald/maxminddb-golang v1.12.0 github.com/safing/jess v0.3.1 - github.com/safing/portbase v0.17.4 + github.com/safing/portbase v0.17.5 github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec - github.com/safing/spn v0.6.19 + github.com/safing/spn v0.6.21 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/cobra v1.7.0 github.com/spkg/zipfs v0.7.1 diff --git a/go.sum b/go.sum index 776f11389..f7b25f3c5 100644 --- a/go.sum +++ b/go.sum @@ -240,12 +240,12 @@ github.com/safing/jess v0.3.1 h1:cMZVhi2whW/YdD98MPLeLIWJndQ7o2QVt2HefQ/ByFA= github.com/safing/jess v0.3.1/go.mod h1:aj73Eot1zm2ETkJuw9hJlIO8bRom52uBbsCHemvlZmA= github.com/safing/portbase v0.15.2/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y= github.com/safing/portbase v0.16.2/go.mod h1:mzNCWqPbO7vIYbbK5PElGbudwd2vx4YPNawymL8Aro8= -github.com/safing/portbase v0.17.4 h1:4RhItvFujwdfLQVfwvB+VYER33AT//Ywv317Vj01TEQ= -github.com/safing/portbase v0.17.4/go.mod h1:suLPSjOTqA7iDLozis5OI7PSw+wqJNT8SLvdBhRPlqI= +github.com/safing/portbase v0.17.5 h1:0gq0tgPLbKlK+xq7WM+Kcutu5HgYIglxBE3QqN5tIAA= +github.com/safing/portbase v0.17.5/go.mod h1:suLPSjOTqA7iDLozis5OI7PSw+wqJNT8SLvdBhRPlqI= github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec h1:oSJY1seobofPwpMoJRkCgXnTwfiQWNfGMCPDfqgAEfg= github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE= -github.com/safing/spn v0.6.19 h1:z4i8hb5FGKjmgSzA4MzJ8mOc0hYp11zgXzujrHwwV5k= -github.com/safing/spn v0.6.19/go.mod h1:LRWLManSXHTViiDqU2qNy3w07auMuadOnVW8wAB/Cgw= +github.com/safing/spn v0.6.21 h1:7LhaEbQ7xrPMETerydpbEAVmLmp+etGJWKnW5b6iI0g= +github.com/safing/spn v0.6.21/go.mod h1:MgWfUDkYqi46A+EcxayLD0tc519KBiVEQ6mfAjHIx/4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0= diff --git a/network/connection.go b/network/connection.go index 083907797..ab3ec3fbf 100644 --- a/network/connection.go +++ b/network/connection.go @@ -488,6 +488,12 @@ func (conn *Connection) GatherConnectionInfo(pkt packet.Packet) (err error) { // Errors are informational and are logged to the context. } + // Only get process and profile with first real packet. + // TODO: Remove when we got full VM/Docker support. + if pkt.InfoOnly() { + return nil + } + // Get Process and Profile. if conn.process == nil { conn.process, err = process.GetProcessWithProfile(pkt.Ctx(), conn.PID) diff --git a/profile/fingerprint.go b/profile/fingerprint.go index 6f5e63893..3f62ba9de 100644 --- a/profile/fingerprint.go +++ b/profile/fingerprint.go @@ -4,6 +4,11 @@ import ( "fmt" "regexp" "strings" + + "golang.org/x/exp/slices" + + "github.com/safing/jess/lhash" + "github.com/safing/portbase/container" ) // # Matching and Scores @@ -57,6 +62,12 @@ type ( Key string // Key must always fully match. Operation string Value string + + // MergedFrom holds the ID of the profile from which this fingerprint was + // merged from. The merged profile should create a new profile ID derived + // from the new fingerprints and add all fingerprints with this field set + // to the originating profile ID + MergedFrom string } // Tag represents a simple key/value kind of tag used in process metadata @@ -347,3 +358,78 @@ func checkMatchStrength(value int) int { } return value } + +const ( + deriveFPKeyIDForItemStart = iota + 1 + deriveFPKeyIDForType + deriveFPKeyIDForKey + deriveFPKeyIDForOperation + deriveFPKeyIDForValue +) + +func deriveProfileID(fps []Fingerprint) string { + // Sort the fingerprints. + sortAndCompactFingerprints(fps) + + // Compile data for hashing. + c := container.New(nil) + c.AppendInt(len(fps)) + for _, fp := range fps { + c.AppendNumber(deriveFPKeyIDForItemStart) + if fp.Type != "" { + c.AppendNumber(deriveFPKeyIDForType) + c.AppendAsBlock([]byte(fp.Type)) + } + if fp.Key != "" { + c.AppendNumber(deriveFPKeyIDForKey) + c.AppendAsBlock([]byte(fp.Key)) + } + if fp.Operation != "" { + c.AppendNumber(deriveFPKeyIDForOperation) + c.AppendAsBlock([]byte(fp.Operation)) + } + if fp.Value != "" { + c.AppendNumber(deriveFPKeyIDForValue) + c.AppendAsBlock([]byte(fp.Value)) + } + } + + // Hash and return. + h := lhash.Digest(lhash.SHA3_256, c.CompileData()) + return h.Base58() +} + +func sortAndCompactFingerprints(fps []Fingerprint) []Fingerprint { + // Sort. + slices.SortFunc[[]Fingerprint, Fingerprint](fps, func(a, b Fingerprint) int { + switch { + case a.Type != b.Type: + return strings.Compare(a.Type, b.Type) + case a.Key != b.Key: + return strings.Compare(a.Key, b.Key) + case a.Operation != b.Operation: + return strings.Compare(a.Operation, b.Operation) + case a.Value != b.Value: + return strings.Compare(a.Value, b.Value) + case a.MergedFrom != b.MergedFrom: + return strings.Compare(a.MergedFrom, b.MergedFrom) + default: + return 0 + } + }) + + // De-duplicate. + // Important: Even if the fingerprint is the same, but MergedFrom is + // different, we need to keep the separate fingerprint, so that new installs + // will cleanly update to the synced state: Auto-generated profiles need to + // be automatically replaced by the merged version. + fps = slices.CompactFunc[[]Fingerprint, Fingerprint](fps, func(a, b Fingerprint) bool { + return a.Type == b.Type && + a.Key == b.Key && + a.Operation == b.Operation && + a.Value == b.Value && + a.MergedFrom == b.MergedFrom + }) + + return fps +} diff --git a/profile/fingerprint_test.go b/profile/fingerprint_test.go new file mode 100644 index 000000000..4857d8bee --- /dev/null +++ b/profile/fingerprint_test.go @@ -0,0 +1,53 @@ +package profile + +import ( + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestDeriveProfileID(t *testing.T) { + t.Parallel() + + fps := []Fingerprint{ + { + Type: FingerprintTypePathID, + Operation: FingerprintOperationEqualsID, + Value: "/sbin/init", + }, + { + Type: FingerprintTypePathID, + Operation: FingerprintOperationPrefixID, + Value: "/", + }, + { + Type: FingerprintTypeEnvID, + Key: "PORTMASTER_PROFILE", + Operation: FingerprintOperationEqualsID, + Value: "TEST-1", + }, + { + Type: FingerprintTypeTagID, + Key: "tag-key-1", + Operation: FingerprintOperationEqualsID, + Value: "tag-key-2", + }, + } + + // Create rand source for shuffling. + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec + + // Test 100 times. + for i := 0; i < 100; i++ { + // Shuffle fingerprints. + rnd.Shuffle(len(fps), func(i, j int) { + fps[i], fps[j] = fps[j], fps[i] + }) + + // Check if fingerprint matches. + id := deriveProfileID(fps) + assert.Equal(t, "PTSRP7rdCnmvdjRoPMTrtjj7qk7PxR1a9YdBWUGwnZXJh2", id) + } +} diff --git a/profile/icon.go b/profile/icon.go new file mode 100644 index 000000000..0f084be59 --- /dev/null +++ b/profile/icon.go @@ -0,0 +1,57 @@ +package profile + +import ( + "strings" + + "golang.org/x/exp/slices" +) + +// Icon describes an icon. +type Icon struct { + Type IconType + Value string +} + +// IconType describes the type of an Icon. +type IconType string + +// Supported icon types. +const ( + IconTypeFile IconType = "path" + IconTypeDatabase IconType = "database" +) + +func (t IconType) sortOrder() int { + switch t { + case IconTypeDatabase: + return 1 + case IconTypeFile: + return 2 + default: + return 100 + } +} + +func sortAndCompactIcons(icons []Icon) []Icon { + // Sort. + slices.SortFunc[[]Icon, Icon](icons, func(a, b Icon) int { + aOrder := a.Type.sortOrder() + bOrder := b.Type.sortOrder() + + switch { + case aOrder != bOrder: + return aOrder - bOrder + case a.Value != b.Value: + return strings.Compare(a.Value, b.Value) + default: + return 0 + } + }) + + // De-duplicate. + icons = slices.CompactFunc[[]Icon, Icon](icons, func(a, b Icon) bool { + return a.Type == b.Type && a.Value == b.Value + }) + + return icons +} diff --git a/profile/merge.go b/profile/merge.go new file mode 100644 index 000000000..581ba07ef --- /dev/null +++ b/profile/merge.go @@ -0,0 +1,84 @@ +package profile + +import ( + "fmt" + "sync" + + "github.com/safing/portbase/database/record" +) + +// MergeProfiles merges multiple profiles into a new one. +// The new profile is saved and returned. +// Only the icon and fingerprints are inherited from other profiles. +// All other information is taken only from the primary profile. +func MergeProfiles(primary *Profile, secondaries ...*Profile) (newProfile *Profile, err error) { + // Fill info from primary profile. + newProfile = &Profile{ + Base: record.Base{}, + RWMutex: sync.RWMutex{}, + ID: "", // Omit ID to derive it from the new fingerprints. + Source: primary.Source, + Name: primary.Name, + Description: primary.Description, + Homepage: primary.Homepage, + UsePresentationPath: false, // Disable presentation path. + SecurityLevel: primary.SecurityLevel, + Config: primary.Config, + } + + // Collect all icons. + newProfile.Icons = make([]Icon, 0, len(secondaries)+1) // Guess the needed space. + newProfile.Icons = append(newProfile.Icons, primary.Icons...) + for _, sp := range secondaries { + newProfile.Icons = append(newProfile.Icons, sp.Icons...) + } + newProfile.Icons = sortAndCompactIcons(newProfile.Icons) + + // Collect all fingerprints. + newProfile.Fingerprints = make([]Fingerprint, 0, len(secondaries)+1) // Guess the needed space. + newProfile.Fingerprints = addFingerprints(newProfile.Fingerprints, primary.Fingerprints, primary.ScopedID()) + for _, sp := range secondaries { + newProfile.Fingerprints = addFingerprints(newProfile.Fingerprints, sp.Fingerprints, sp.ScopedID()) + } + newProfile.Fingerprints = sortAndCompactFingerprints(newProfile.Fingerprints) + + // Save new profile. + newProfile = New(newProfile) + err = newProfile.Save() + if err != nil { + return nil, fmt.Errorf("failed to save merged profile: %w", err) + } + // FIXME: Should we ... ? + // newProfile.updateMetadata() + // newProfile.updateMetadataFromSystem() + + // Delete all previous profiles. + // FIXME: + /* + primary.Meta().Delete() + // Set as outdated and remove from active profiles. + // Signify that profile was deleted and save for sync. + for _, sp := range secondaries { + sp.Meta().Delete() + // Set as outdated and remove from active profiles. + // Signify that profile was deleted and save for sync. + } + */ + + return newProfile, nil +} + +func addFingerprints(existing, add []Fingerprint, from string) []Fingerprint { + // Copy all fingerprints and add the they are from. + for _, addFP := range add { + existing = append(existing, Fingerprint{ + Type: addFP.Type, + Key: addFP.Key, + Operation: addFP.Operation, + Value: addFP.Value, + MergedFrom: from, + }) + } + + return existing +} diff --git a/profile/migrations.go b/profile/migrations.go index 5db789580..fa19a8ccf 100644 --- a/profile/migrations.go +++ b/profile/migrations.go @@ -2,6 +2,8 @@ package profile import ( "context" + "fmt" + "regexp" "github.com/hashicorp/go-version" @@ -25,6 +27,16 @@ func registerMigrations() error { Version: "v0.9.9", MigrateFunc: migrateLinkedPath, }, + migration.Migration{ + Description: "Migrate from Icon Fields to Icon List", + Version: "v1.4.7", + MigrateFunc: migrateIcons, + }, + // migration.Migration{ + // Description: "Migrate from random profile IDs to fingerprint-derived IDs", + // Version: "v1.5.1", + // MigrateFunc: migrateToDerivedIDs, + // }, ) } @@ -97,3 +109,154 @@ func migrateLinkedPath(ctx context.Context, _, to *version.Version, db *database return nil } + +func migrateIcons(ctx context.Context, _, to *version.Version, db *database.Interface) error { + // Get iterator over all profiles. + it, err := db.Query(query.New(profilesDBPath)) + if err != nil { + log.Tracer(ctx).Errorf("profile: failed to migrate from icon fields: failed to start query: %s", err) + return nil + } + + // Migrate all profiles. + var ( + lastErr error + failed int + total int + ) + for r := range it.Next { + // Parse profile. + profile, err := EnsureProfile(r) + if err != nil { + log.Tracer(ctx).Debugf("profiles: failed to parse profile %s for migration: %s", r.Key(), err) + continue + } + + // Skip if there is no (valid) icon defined or the icon list is already populated. + if profile.Icon == "" || profile.IconType == "" || len(profile.Icons) > 0 { + continue + } + + // Migrate to icon list. + profile.Icons = []Icon{{ + Type: profile.IconType, + Value: profile.Icon, + }} + + // Save back to DB. + err = db.Put(profile) + if err != nil { + failed++ + lastErr = err + log.Tracer(ctx).Debugf("profiles: failed to save profile %s after migration: %s", r.Key(), err) + } else { + log.Tracer(ctx).Tracef("profiles: migrated profile %s to %s", r.Key(), to) + } + total++ + } + + // Check if there was an error while iterating. + if err := it.Err(); err != nil { + log.Tracer(ctx).Errorf("profile: failed to migrate from icon fields: failed to iterate over profiles for migration: %s", err) + } + + // Log migration failure and try again next time. + if lastErr != nil { + // Normally, an icon migration would not be such a big error, but this is a test + // run for the profile IDs and we absolutely need to know if anything went wrong. + module.Error( + "migration-failed", + "Profile Migration Failed", + fmt.Sprintf("Failed to migrate icons of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr), + ) + return fmt.Errorf("failed to migrate %d profiles (out of %d pending) - last error: %w", failed, total, lastErr) + } + + return lastErr +} + +var randomUUIDRegex = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +func migrateToDerivedIDs(ctx context.Context, _, to *version.Version, db *database.Interface) error { + var profilesToDelete []string //nolint:prealloc // We don't know how many profiles there are. + + // Get iterator over all profiles. + it, err := db.Query(query.New(profilesDBPath)) + if err != nil { + log.Tracer(ctx).Errorf("profile: failed to migrate to derived profile IDs: failed to start query: %s", err) + return nil + } + + // Migrate all profiles. + var ( + lastErr error + failed int + total int + ) + for r := range it.Next { + // Parse profile. + profile, err := EnsureProfile(r) + if err != nil { + log.Tracer(ctx).Debugf("profiles: failed to parse profile %s for migration: %s", r.Key(), err) + continue + } + + // Skip if the ID does not look like a random UUID. + if !randomUUIDRegex.MatchString(profile.ID) { + continue + } + + // Generate new ID. + oldScopedID := profile.ScopedID() + newID := deriveProfileID(profile.Fingerprints) + + // If they match, skip migration for this profile. + if profile.ID == newID { + continue + } + + // Reset key. + profile.ResetKey() + // Set new ID and rebuild the key. + profile.ID = newID + profile.makeKey() + + // Save back to DB. + err = db.Put(profile) + if err != nil { + failed++ + lastErr = err + log.Tracer(ctx).Debugf("profiles: failed to save profile %s after migration: %s", r.Key(), err) + } else { + log.Tracer(ctx).Tracef("profiles: migrated profile %s to %s", r.Key(), to) + + // Add old ID to profiles that we need to delete. + profilesToDelete = append(profilesToDelete, oldScopedID) + } + total++ + } + + // Check if there was an error while iterating. + if err := it.Err(); err != nil { + log.Tracer(ctx).Errorf("profile: failed to migrate to derived profile IDs: failed to iterate over profiles for migration: %s", err) + } + + // Delete old migrated profiles. + for _, scopedID := range profilesToDelete { + if err := db.Delete(profilesDBPath + scopedID); err != nil { + log.Tracer(ctx).Errorf("profile: failed to delete old profile %s during migration: %s", scopedID, err) + } + } + + // Log migration failure and try again next time. + if lastErr != nil { + module.Error( + "migration-failed", + "Profile Migration Failed", + fmt.Sprintf("Failed to migrate profile IDs of %d profiles (out of %d pending). The last error was: %s\n\nPlease restart Portmaster to try the migration again.", failed, total, lastErr), + ) + return fmt.Errorf("failed to migrate %d profiles (out of %d pending) - last error: %w", failed, total, lastErr) + } + + return nil +} diff --git a/profile/module.go b/profile/module.go index 7a5fbef49..c41c90046 100644 --- a/profile/module.go +++ b/profile/module.go @@ -48,7 +48,7 @@ func start() error { } if err := migrations.Migrate(module.Ctx); err != nil { - return err + log.Errorf("profile: migrations failed: %s", err) } err := registerValidationDBHook() diff --git a/profile/profile.go b/profile/profile.go index 42a39738a..c57c52d7d 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -37,17 +37,6 @@ const ( DefaultActionPermit uint8 = 3 ) -// iconType describes the type of the Icon property -// of a profile. -type iconType string - -// Supported icon types. -const ( - IconTypeFile iconType = "path" - IconTypeDatabase iconType = "database" - IconTypeBlob iconType = "blob" -) - // Profile is used to predefine a security profile for applications. type Profile struct { //nolint:maligned // not worth the effort record.Base @@ -73,12 +62,16 @@ type Profile struct { //nolint:maligned // not worth the effort // Homepage may refer to the website of the application // vendor. Homepage string - // Icon holds the icon of the application. The value + + // Deprecated: Icon holds the icon of the application. The value // may either be a filepath, a database key or a blob URL. // See IconType for more information. Icon string - // IconType describes the type of the Icon property. - IconType iconType + // Deprecated: IconType describes the type of the Icon property. + IconType IconType + // Icons holds a list of icons to represent the application. + Icons []Icon + // Deprecated: LinkedPath used to point to the executableis this // profile was created for. // Until removed, it will be added to the Fingerprints as an exact path match. @@ -265,9 +258,16 @@ func New(profile *Profile) *Profile { profile.Config = make(map[string]interface{}) } - // Generate random ID if none is given. + // Generate ID if none is given. if profile.ID == "" { - profile.ID = utils.RandomUUID("").String() + if len(profile.Fingerprints) > 0 { + // Derive from fingerprints. + profile.ID = deriveProfileID(profile.Fingerprints) + } else { + // Generate random ID as fallback. + log.Warningf("profile: creating new profile without fingerprints to derive ID from") + profile.ID = utils.RandomUUID("").String() + } } // Make key from ID and source. diff --git a/resolver/scopes.go b/resolver/scopes.go index 93a7b97cb..67195f1db 100644 --- a/resolver/scopes.go +++ b/resolver/scopes.go @@ -171,8 +171,11 @@ func GetResolversInScope(ctx context.Context, q *Query) (selected []*Resolver, p // Special connectivity domains if netenv.IsConnectivityDomain(q.FQDN) && len(systemResolvers) > 0 { - // Do not do compliance checks for connectivity domains. - selected = append(selected, systemResolvers...) // dhcp assigned resolvers + selected = addResolvers(ctx, q, selected, systemResolvers) + if len(selected) == 0 { + selected = addResolvers(ctx, q, selected, localResolvers) + selected = addResolvers(ctx, q, selected, globalResolvers) + } return selected, ServerSourceOperatingSystem, false }