feat(core): adds storeFile to save encrypted profiles to disk and updates auth to propagate tlsNoVerify

.github/scripts/verify-checksums.sh

@@ -2,32 +2,60 @@
 
 # Check if the required arguments are provided
 if [ $# -ne 2 ]; then
-    echo "Usage: $0 <outputDir> <checksumFile>"
+    echo "Usage: $0 <output_directory> <checksum_file>"
     exit 1
 fi
 
-echo "Verifying checksums..."
-# Location of the checksum file
-checksumFile=$1/$2
-outputDir=$1
+# Assign arguments to variables
+output_dir="$1"
+checksum_file="$2"
+checksum_path="${output_dir}/${checksum_file}"  # Full path to the checksum file
+lock_file="${checksum_path}.lock"  # Append .lock to the full path of the checksum file
+
+# Ensure the checksum file exists
+if [ ! -f "$checksum_path" ]; then
+    echo "ERROR: Checksum file $checksum_path does not exist."
+    exit 1
+fi
 
-echo "Looking for checksum file: $checksumFile"
-test -f "$checksumFile" || { echo "ERROR: Checksum file not found!"; exit 1; }
+# Wait for the lock file to be available for reading
+exec 200<"$lock_file"  # Open lock file descriptor for reading
+flock -s 200           # Acquire shared lock (will wait if exclusive lock is held)
+
+echo "Verifying checksums..."
+echo "Looking for checksum file: $checksum_path"
 
 # Iterate over each line in the checksum file
 while read -r line; do
-  # Extract the expected checksum and filename from each line
-  read -ra ADDR <<< "$line" # Read the line into an array
-  expectedChecksum="${ADDR[0]}"
-  fileName="${ADDR[2]}"
-
-  # Calculate the actual checksum of the file
-  actualChecksum=$(shasum -a 256 "$outputDir/$fileName" | awk '{print $1}')
-
-  # Compare the expected checksum with the actual checksum
-  if [ "$expectedChecksum" == "$actualChecksum" ]; then
-    echo "SUCCESS: Checksum for $fileName is valid."
-  else
-    echo "ERROR: Checksum for $fileName does not match."
-  fi
-done < "$checksumFile"
+    # Extract checksum and filename from the line
+    expected_checksum=$(echo "$line" | awk '{print $1}')
+    filename=$(echo "$line" | awk '{print $2}')
+
+    # Construct the full path to the file
+    file_path="$output_dir/$filename"
+
+    # Check if the file exists
+    if [ ! -f "$file_path" ]; then
+        echo "ERROR: File $filename not found in $output_dir"
+        continue
+    fi
+
+    # Calculate the actual checksum of the file
+    actual_checksum=$(shasum -a 256 "$file_path" | awk '{print $1}')
+
+    # Compare the expected and actual checksums
+    if [ "$expected_checksum" != "$actual_checksum" ]; then
+        echo "ERROR: Checksum for $filename does not match."
+    else
+        echo "Checksum for $filename is correct."
+    fi
+done < "$checksum_path"
+
+# Release the lock and close the lock file descriptor
+flock -u 200
+exec 200>&-
+
+# Clean up the lock file
+rm -f "$lock_file"
+
+echo "Checksum verification completed."

.github/scripts/zip-builds.sh

@@ -16,23 +16,41 @@ mkdir -p "$output_dir"
 
 # Create a checksums file
 checksums_file="$output_dir/${build_semver}_checksums.txt"
-touch $checksums_file
+touch "$checksums_file"
+
+# Define a lock file for parallel-safe writing to checksums
+checksums_lockfile="${checksums_file}.lock"
 
 # Iterate over each binary file
 for binary_file in "$binary_dir"/*; do
-    compressed=""
-    if [[ $binary_file == *.exe ]]; then
-        # If the file is a Windows binary, zip it
-        filename=$(basename "$binary_file")
-        compressed="${filename%.exe}.zip"
-        zip -j "$output_dir/$compressed" "$binary_file"
-    else
-        # For other binaries, tar and gzip them
-        filename=$(basename "$binary_file")
-        compressed="${filename}.tar.gz"
-        tar -czf "$output_dir/$compressed" "$binary_file"
-    fi
-
-    # Append checksums to the file
-    echo "$(cat "$output_dir/$compressed" | shasum -a 256) $compressed" >> $checksums_file
-done
+    (
+        compressed=""
+        if [[ $binary_file == *.exe ]]; then
+            # If the file is a Windows binary, zip it
+            filename=$(basename "$binary_file")
+            compressed="${filename%.exe}.zip"
+            zip -j "$output_dir/$compressed" "$binary_file"
+        else
+            # For other binaries, tar and gzip them
+            filename=$(basename "$binary_file")
+            compressed="${filename}.tar.gz"
+            tar -czf "$output_dir/$compressed" "$binary_file"
+        fi
+
+        # Compute checksum and append it to the checksums file using a lock
+        checksum="$(shasum -a 256 "$output_dir/$compressed" | awk '{print $1}')"
+        (
+            flock -x 200
+            echo "$checksum $compressed" >> "$checksums_file"
+        ) 200>"$checksums_lockfile"
+
+    ) &
+done
+
+# Echo message indicating background tasks are running
+echo "All zip and tar processes started. Waiting for them to finish..."
+
+# Wait for all background processes to complete
+wait
+
+echo "All compression and checksum operations completed."

Makefile

@@ -58,10 +58,10 @@ build-%:
 	go build $(GO_BUILD_FLAGS) \
 		-o $(GO_BUILD_PREFIX)-$(word 1,$(subst -, ,$*))-$(word 2,$(subst -, ,$*))$(word 3,$(subst -, ,$*))
 
-zip-builds:
+zip-builds: $(addprefix build-,$(PLATFORMS))
 	./.github/scripts/zip-builds.sh $(BINARY_NAME)-$(CURR_VERSION) $(TARGET_DIR) $(OUTPUT_DIR)
 
-verify-checksums:
+verify-checksums: zip-builds
 	./.github/scripts/verify-checksums.sh $(OUTPUT_DIR) $(BINARY_NAME)-$(CURR_VERSION)_checksums.txt 
 
 # Target for running the project (adjust as necessary for your project)
@@ -93,3 +93,4 @@ test-bats: build-test
 .PHONY: clean
 clean:
 	rm -rf $(TARGET_DIR)
+	rm -rf $(OUTPUT_DIR)

cmd/auth-login.go

@@ -13,11 +13,16 @@ func auth_codeLogin(cmd *cobra.Command, args []string) {
 	_, cp := InitProfile(c, false)
 
 	c.Print("Initiating login...")
+
+	// Use profile values as defaults, with command-line overrides
+	tlsNoVerify := c.FlagHelper.GetOptionalBoolWithDefault("tls-no-verify", cp.GetTLSNoVerify())
+	clientId := c.FlagHelper.GetOptionalStringWithDefault("client-id", cp.GetAuthCredentials().ClientId)
+
 	tok, publicClientID, err := auth.LoginWithPKCE(
 		cmd.Context(),
 		cp.GetEndpoint(),
-		c.FlagHelper.GetOptionalString("client-id"),
-		c.FlagHelper.GetOptionalBool("tls-no-verify"),
+		clientId,
+		tlsNoVerify,
 	)
 	if err != nil {
 		c.Println("failed")

cmd/root.go

@@ -279,5 +279,11 @@ func init() {
 		rootCmd.GetDocFlag("with-access-token").Default,
 		rootCmd.GetDocFlag("with-access-token").Description,
 	)
+
+	RootCmd.PersistentFlags().String(
+		rootCmd.GetDocFlag("profile-driver").Name,
+		rootCmd.GetDocFlag("profile-driver").Default,
+		rootCmd.GetDocFlag("profile-driver").Description,
+	)
 	RootCmd.AddGroup(&cobra.Group{ID: TDF})
 }

docs/man/_index.md

@@ -39,4 +39,11 @@ command:
     - name: debug
       description: enable debug output
       default: false
+    - name: profile-driver
+      description: storage driver for managing profiles
+      enum:
+        - keyring
+        - in-memory
+        - file
+      default: file
 ---

e2e/profile.bats

@@ -65,10 +65,7 @@ teardown() {
 @test "profile create" {
   run_otdfctl profile create test http://localhost:8080
   assert_output --regexp "Creating profile .* ok"
-
-  run_otdfctl profile create test localhost:8080
-  assert_output --regexp "Failed .* invalid scheme"
-
+
   # TODO figure out how to test the case where the profile already exists
 }
 

go.mod

@@ -15,9 +15,9 @@ require (
 	github.com/go-jose/go-jose/v3 v3.0.3
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/uuid v1.6.0
-	github.com/opentdf/platform/lib/flattening v0.1.1
-	github.com/opentdf/platform/protocol/go v0.2.20
-	github.com/opentdf/platform/sdk v0.3.19
+	github.com/opentdf/platform/lib/flattening v0.1.2
+	github.com/opentdf/platform/protocol/go v0.2.22
+	github.com/opentdf/platform/sdk v0.3.20
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
 	github.com/stretchr/testify v1.9.0
@@ -86,7 +86,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sahilm/fuzzy v0.1.1-0.20230530133925-c48e322e2a8f // indirect
@@ -109,9 +108,9 @@ require (
 	go.opentelemetry.io/otel/metric v1.29.0 // indirect
 	go.opentelemetry.io/otel/trace v1.29.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/crypto v0.26.0 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.27.0 // indirect
 	golang.org/x/text v0.17.0 // indirect

go.sum

@@ -215,22 +215,16 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
-github.com/opentdf/platform/lib/fixtures v0.2.7 h1:2LxWmLBBISONVJnVDH8yMsV72VHQyirua0DwDBBoq+g=
-github.com/opentdf/platform/lib/fixtures v0.2.7/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8=
-github.com/opentdf/platform/lib/flattening v0.1.1 h1:la1f6PcRsc+yLH8+9UEr0ux6IRKu+6+oMaMVt05+8HU=
-github.com/opentdf/platform/lib/flattening v0.1.1/go.mod h1:eyG7pe5UZlV+GI5/CymQD3xTAJxNhnP9M4QnBzaad1M=
+github.com/opentdf/platform/lib/fixtures v0.2.8 h1:lGYrMnbORtU62lxsJi8qPsxjFuNIkc4Dop8rVkH6pD0=
+github.com/opentdf/platform/lib/fixtures v0.2.8/go.mod h1:8yCSe+oUzW9jbM573r9qgE68rjwDMNzktObiGVsO/W8=
+github.com/opentdf/platform/lib/flattening v0.1.2 h1:7/fUlBY08PR6UItfVU2CVF5rcCxf5oZZ4MGLABj4NAU=
+github.com/opentdf/platform/lib/flattening v0.1.2/go.mod h1:Gs/T+6FGZKk9OAdz2Jf1R8CTGeNRYrq1lZGDeYT3hrY=
 github.com/opentdf/platform/lib/ocrypto v0.1.6 h1:rd4ctCZOE/c3qDJORtkSK9tw6dEXb+jbJXRRk4LcxII=
 github.com/opentdf/platform/lib/ocrypto v0.1.6/go.mod h1:ne+l8Q922OdzA0xesK3XJmfECBnn5vLSGYU3/3OhiHM=
-github.com/opentdf/platform/protocol/go v0.2.18 h1:s+TVZkOPGCzy7WyObtJWJNaFeOGDUTuSmAsq3omvugY=
-github.com/opentdf/platform/protocol/go v0.2.18/go.mod h1:WqDcnFQJb0v8ivRQPidbehcL8ils5ZSZYXkuv0nyvsI=
-github.com/opentdf/platform/protocol/go v0.2.20 h1:FPU1ZcXvPm/QeE2nqgbD/HMTOCICQSD0DoncQbAZ1ws=
-github.com/opentdf/platform/protocol/go v0.2.20/go.mod h1:TWIuf387VeR3q0TL4nAMKQTWEqqID+8Yjao76EX9Dto=
-github.com/opentdf/platform/sdk v0.3.17 h1:Uo/kTMneB18i0gZNfTRtvw34bGLFUc8BEnA/BMK0VVs=
-github.com/opentdf/platform/sdk v0.3.17/go.mod h1:c2+nrsRLvLf2OOryXnNy0iGZN/TScc21Pul7uqKVXIs=
-github.com/opentdf/platform/sdk v0.3.18 h1:IY6fNrOfQD9lF/hZp9ewZsH0PMuLe17HlSE1A5kyIWc=
-github.com/opentdf/platform/sdk v0.3.18/go.mod h1:u+XZhVRsMq5blukCFCHcjk6HLCp4Y5mmIQu7GhtKQ3E=
-github.com/opentdf/platform/sdk v0.3.19 h1:4Ign6HPrxOH6ZllLO/cI6joSuqz8CqPlpxpTKunpMQs=
-github.com/opentdf/platform/sdk v0.3.19/go.mod h1:u+XZhVRsMq5blukCFCHcjk6HLCp4Y5mmIQu7GhtKQ3E=
+github.com/opentdf/platform/protocol/go v0.2.22 h1:C/jjtwu5yTon8g0ewuN29QE7VXSQHyb2dx9W0U6Oqok=
+github.com/opentdf/platform/protocol/go v0.2.22/go.mod h1:skpOCVuWSjUHazLKOkh3nSB057OB4sHICe7MpmJY9KU=
+github.com/opentdf/platform/sdk v0.3.20 h1:zyBAZLhQaIv4X2twyPbmbdBd9Vc1vsTwxr1BIuESJWg=
+github.com/opentdf/platform/sdk v0.3.20/go.mod h1:O4tyqjK9sJwp+6jUeiJjECe9TQfqaD1kTr6wgsRxkWc=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
@@ -338,8 +332,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ=
 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -351,8 +345,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
 golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
 golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

pkg/auth/auth.go

@@ -214,7 +214,7 @@ const (
 
 // Facilitates an auth code PKCE flow to obtain OIDC tokens.
 // Spawns a local server to handle the callback and opens a browser window in each respective OS.
-func Login(ctx context.Context, platformEndpoint, tokenURL, authURL, publicClientID string) (*oauth2.Token, error) {
+func Login(ctx context.Context, platformEndpoint, tokenURL, authURL, publicClientID string, tlsNoVerify bool) (*oauth2.Token, error) {
 	// Generate random hash and encryption keys for cookie handling
 	hashKey := make([]byte, keyLength)
 	encryptKey := make([]byte, keyLength)
@@ -239,9 +239,16 @@ func Login(ctx context.Context, platformEndpoint, tokenURL, authURL, publicClien
 		},
 	}
 
-	cookiehandler := httphelper.NewCookieHandler(hashKey, encryptKey)
+	var cookieOpts []httphelper.CookieHandlerOpt
+	if tlsNoVerify {
+		cookieOpts = append(cookieOpts, httphelper.WithUnsecure())
+	}
+
+	cookiehandler := httphelper.NewCookieHandler(hashKey, encryptKey, cookieOpts...)
 
 	relyingParty, err := oidcrp.NewRelyingPartyOAuth(conf,
+		// respect tlsNoVerify
+		oidcrp.WithHTTPClient(utils.NewHttpClient(tlsNoVerify)),
 		// allow cookie handling for PKCE
 		oidcrp.WithCookieHandler(cookiehandler),
 		// use PKCE
@@ -271,7 +278,7 @@ func LoginWithPKCE(ctx context.Context, host, publicClientID string, tlsNoVerify
 		return nil, "", fmt.Errorf("failed to get platform configuration: %w", err)
 	}
 
-	tok, err := Login(ctx, host, pc.tokenEndpoint, pc.authzEndpoint, pc.publicClientID)
+	tok, err := Login(ctx, host, pc.tokenEndpoint, pc.authzEndpoint, pc.publicClientID, tlsNoVerify)
 	if err != nil {
 		return nil, "", fmt.Errorf("failed to login: %w", err)
 	}

pkg/cli/flagValues.go

@@ -52,11 +52,18 @@ func (f flagHelper) GetOptionalID(idFlag string) string {
 }
 
 func (f flagHelper) GetOptionalString(flag string) string {
-	p := f.cmd.Flag(flag)
-	if p == nil {
-		return ""
+	return f.GetOptionalStringWithDefault(flag, "")
+}
+
+// GetOptionalStringWithDefault retrieves a string flag, or returns the default value if the flag is not set
+func (f flagHelper) GetOptionalStringWithDefault(flagName string, defaultValue string) string {
+	if f.cmd.Flags().Changed(flagName) {
+		value, err := f.cmd.Flags().GetString(flagName)
+		if err == nil {
+			return value
+		}
 	}
-	return p.Value.String()
+	return defaultValue
 }
 
 func (f flagHelper) GetStringSlice(flag string, v []string, opts FlagsStringSliceOptions) []string {
@@ -82,8 +89,18 @@ func (f flagHelper) GetRequiredInt32(flag string) int32 {
 }
 
 func (f flagHelper) GetOptionalBool(flag string) bool {
-	v, _ := f.cmd.Flags().GetBool(flag)
-	return v
+	return f.GetOptionalBoolWithDefault(flag, false)
+}
+
+// GetOptionalBoolWithDefault retrieves a boolean flag, or returns the default value if the flag is not set
+func (f flagHelper) GetOptionalBoolWithDefault(flagName string, defaultValue bool) bool {
+	if f.cmd.Flags().Changed(flagName) {
+		value, err := f.cmd.Flags().GetBool(flagName)
+		if err == nil {
+			return value
+		}
+	}
+	return defaultValue
 }
 
 func (f flagHelper) GetRequiredBool(flag string) bool {