rsc: Support db only uri scheme for output files (#1638)

* Support db only uri scheme for output files * Fix lint * Fix uri path substringing * address comments * format * Update remote_cache_api.wake * Update share/wake/lib/system/remote_cache_api.wake Co-authored-by: Sam May <[email protected]> --------- Co-authored-by: Ashley Coleman <[email protected]> Co-authored-by: Sam May <[email protected]>
sifive · Aug 30, 2024 · 2bc8170 · 2bc8170
1 parent 462ae15
commit 2bc8170
Showing 1 changed file with 60 additions and 50 deletions.
diff --git a/share/wake/lib/system/remote_cache_api.wake b/share/wake/lib/system/remote_cache_api.wake
@@ -533,78 +533,88 @@ export target rscApiCheckAuthorization (api: RemoteCacheApi): Result Unit Error
         Some x -> failWithError "Invalid auth key. Status code: {str x}"
         None -> failWithError "Invalid auth key. Unable to determine status code"
 
-# rscApiGetStringBlob: Downloads a blob and returns the contents as a string 
-#
-# ```
-#  rscApiGetStringBlob (RemoteCacheBlob "asdf" "https://...") = Pass "foo\nbar\nbat" 
-# ```
-export def rscApiGetStringBlob ((CacheSearchBlob _ uri): CacheSearchBlob): Result String Error =
+# Converts db only blob uris into actual data
+# maps path = "%46%6F%6F" to content = "foo"
+def resolveDbScheme _scheme path =
+    # If path is empty we have the empty string
+    require True = path.strlen > 0
+    else Pass ""
+
+    # If not empty then it must as least be %00
+    require True = path.strlen >= 3
+    else failWithError "rsc: Invalid db path: '{path}'"
+
+    require "", byteStrs = tokenize `%` path
+    else failWithError "rsc: Failed to extract bytes from db path: '{path}'"
+
+    require Some bytes =
+        byteStrs
+        | map (intbase 16)
+        | findNone
+    else failWithError "rsc: Failed to parse bytes strings into bytes from db path: '{path}'"
+
+    bytes
+    | map integerToByte
+    | cat
+    | Pass
+
+# Dispatches the content resolution for uri based on its scheme
+def resolveUriResponse (dbSchemeFn: (String => String => Result a Error)) (fileSchemeFn: (String => String => Result a Error)) (uri: String) =
     require scheme, path, Nil = extract `([a-zA-Z][a-zA-Z0-9+.-]*)://(.*)` uri
     else failWithError "rsc: uri has unexpected format: '{uri}'"
 
-    # maps path = "%46%6F%6F" to content = "foo"
-    def dbScheme _scheme path =
-        # If path is empty we have the empty string
-        require True = path.strlen > 0
-        else Pass ""
-
-        # If not empty then it must as least be %00
-        require True = path.strlen >= 3
-        else failWithError "rsc: Invalid db path: '{path}'"
-
-        require "", byteStrs = tokenize `%` path
-        else failWithError "rsc: Failed to extract bytes from db path: '{path}'"
-
-        require Some bytes =
-            byteStrs
-            | map (intbase 16)
-            | findNone
-        else failWithError "rsc: Failed to parse bytes strings into bytes from db path: '{path}'"
-
-        bytes
-        | map integerToByte
-        | cat
-        | Pass
-
-    def fileScheme scheme path =
-        require Pass response =
-            buildHttpRequest "{scheme}://{path}"
-            | setMethod HttpMethodGet
-            | makeRequest
-
-        Pass response.getHttpResponseBody
-
-    def unsupportedScheme scheme path =
+    def unsupportedSchemeFn s p =
         failWithError "rsc: scheme '{scheme}' from uri '{uri}' is not supported"
 
     def schemeFn = match scheme
-        "db" -> dbScheme
-        "file" -> fileScheme
-        "http" -> fileScheme
-        "https" -> fileScheme
-        _ -> unsupportedScheme
+        "db" -> dbSchemeFn
+        "file" -> fileSchemeFn
+        "http" -> fileSchemeFn
+        "https" -> fileSchemeFn
+        _ -> unsupportedSchemeFn
 
     schemeFn scheme path
 
+# rscApiGetStringBlob: Downloads a blob and returns the contents as a string 
+#
+# ```
+#  rscApiGetStringBlob (RemoteCacheBlob "asdf" "https://...") = Pass "foo\nbar\nbat" 
+# ```
+export def rscApiGetStringBlob ((CacheSearchBlob _ uri): CacheSearchBlob): Result String Error =
+    def resolveStringFileScheme scheme path =
+        buildHttpRequest "{scheme}://{path}"
+        | setMethod HttpMethodGet
+        | makeRequest
+        |< getHttpResponseBody
+
+    uri
+    | resolveUriResponse resolveDbScheme resolveStringFileScheme
+
 # rscApiGetFileBlob: Downloads a blob to *path* with *mode* permisssions and return *path*
 #
 # ```
 #  rscApiGetFileBlob (RemoteCacheBlob "asdf" "https://...") "foo/bar" 0644 = Pass "foo/bar" 
 # ```
 export def rscApiGetFileBlob ((CacheSearchBlob _ uri): CacheSearchBlob) (path: String) (mode: Integer): Result String Error =
-    require Pass downloadPath =
-        buildHttpRequest uri
+    def resolveDbSchemeToFile scheme path =
+        resolveDbScheme scheme path
+        |> writeTempFile "rsc.output_file.blob"
+
+    def resolveBlobFileScheme scheme path =
+        buildHttpRequest "{scheme}://{path}"
         | setMethod HttpMethodGet
         | makeBinaryRequest
 
+    require Pass blobPath = (uri | resolveUriResponse resolveDbSchemeToFile resolveBlobFileScheme)
+
     def fixupScript =
         """
         # 1. No failures are acceptable
         set -e
         # 2. Remove the old rsctmp file if it exists
         rm -f '%{path}.rsctmp'
         # 3. Hardlink the file to not use 2x disk space
-        cp -l %{downloadPath.getPathName} '%{path}.rsctmp'
+        cp -l %{blobPath.getPathName} '%{path}.rsctmp'
         # 4. Set the permissions as specified
         chmod %{mode | strOctal} '%{path}.rsctmp'
         # 5. If the file was previously created with the exact inode by the rsc but not cleaned up
@@ -615,8 +625,8 @@ export def rscApiGetFileBlob ((CacheSearchBlob _ uri): CacheSearchBlob) (path: S
         """
 
     def job =
-        makeShellPlan fixupScript (downloadPath, Nil)
-        | setPlanLabel "rsc: fixup blob {path} from {downloadPath.getPathName}"
+        makeShellPlan fixupScript (blobPath, Nil)
+        | setPlanLabel "rsc: fixup blob {path} from {blobPath.getPathName}"
         | setPlanPersistence Once
         # We need to copy the file over another jobs output space so:
         #   - We must run with the localRunner
@@ -627,7 +637,7 @@ export def rscApiGetFileBlob ((CacheSearchBlob _ uri): CacheSearchBlob) (path: S
         | setJobInspectVisibilityHidden
 
     require True = job.isJobOk
-    else failWithError "Failed to cleanup downloaded file {path} from {downloadPath.getPathName}"
+    else failWithError "Failed to cleanup downloaded file {path} from {blobPath.getPathName}"
 
     Pass path