From 1843d746e08c685d31cd4366db2e6006b998cee1 Mon Sep 17 00:00:00 2001 From: andy Date: Fri, 19 Apr 2024 23:46:36 -0400 Subject: [PATCH] feat: adding httpResolver - added httpAsset resolver that while resolving the first asset with an http path, will save that baseUrl and search for assets relative to that path. Calls are made in JS through EM_JS calls, and using -sASYNCIFY allows this call to JS to be synchronous from the C++ side. --- pxr/usdImaging/hdEmscripten/CMakeLists.txt | 9 +- .../hdEmscripten/httpResolver/CMakeLists.txt | 14 ++ .../hdEmscripten/httpResolver/plugInfo.json | 18 ++ .../hdEmscripten/httpResolver/resolver.cpp | 225 ++++++++++++++++++ .../hdEmscripten/httpResolver/resolver.h | 29 +++ 5 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 pxr/usdImaging/hdEmscripten/httpResolver/CMakeLists.txt create mode 100644 pxr/usdImaging/hdEmscripten/httpResolver/plugInfo.json create mode 100644 pxr/usdImaging/hdEmscripten/httpResolver/resolver.cpp create mode 100644 pxr/usdImaging/hdEmscripten/httpResolver/resolver.h diff --git a/pxr/usdImaging/hdEmscripten/CMakeLists.txt b/pxr/usdImaging/hdEmscripten/CMakeLists.txt index 93d3634846..5f5a1ede7c 100644 --- a/pxr/usdImaging/hdEmscripten/CMakeLists.txt +++ b/pxr/usdImaging/hdEmscripten/CMakeLists.txt @@ -1,5 +1,6 @@ set(PXR_PREFIX pxr/hdEmscripten) set(PXR_PACKAGE hdEmscripten) +add_subdirectory(httpResolver) if (NOT PXR_ENABLE_JS_SUPPORT) return() @@ -13,6 +14,7 @@ pxr_library(hdEmscripten usd usdUtils sdf + httpResolver PUBLIC_CLASSES webRenderDelegate @@ -22,7 +24,7 @@ pxr_library(hdEmscripten ) -target_link_options(hdEmscripten_internal PRIVATE "SHELL: -sEXPORT_NAME=getUsdModule -sMODULARIZE=1 -lembind -sFORCE_FILESYSTEM=1") +target_link_options(hdEmscripten_internal PRIVATE "SHELL: -Os -g0 -sEXPORT_NAME=getUsdModule -sMODULARIZE=1 -sFORCE_FILESYSTEM=1 -sASYNCIFY") target_compile_options(hdEmscripten_internal PRIVATE "SHELL: -lembind") set(RESOURCE_PARAMETERS "") @@ -46,6 +48,7 @@ add_resources(usdLux) add_resources(ar) add_resources(usdGeom) add_resources(ndr) +add_resources(httpResolver) list(APPEND RESOURCE_PARAMETERS "--preload-file ${PROJECT_BINARY_DIR}/plugins_plugInfo.json@/usd/plugInfo.json") @@ -71,6 +74,7 @@ pxr_cpp_bin(${BINDINGS_NAME} LIBRARIES hdEmscripten usdImaging + httpResolver ${RESOURCE_PARAMETERS} ) @@ -78,8 +82,7 @@ set_target_properties(${BINDINGS_NAME} PROPERTIES SUFFIX ".js" ) -target_link_options(${BINDINGS_NAME} PRIVATE "SHELL:-sEXPORT_NAME=getUsdModule -sMODULARIZE=1 -lembind -sFORCE_FILESYSTEM=1") -target_compile_options(${BINDINGS_NAME} PRIVATE "SHELL: -lembind") +target_link_options(${BINDINGS_NAME} PRIVATE "SHELL: -Os -g0 -sEXPORT_NAME=getUsdModule -sMODULARIZE=1 -lembind -sFORCE_FILESYSTEM=1 -sASYNCIFY") install( FILES diff --git a/pxr/usdImaging/hdEmscripten/httpResolver/CMakeLists.txt b/pxr/usdImaging/hdEmscripten/httpResolver/CMakeLists.txt new file mode 100644 index 0000000000..43ab89c28a --- /dev/null +++ b/pxr/usdImaging/hdEmscripten/httpResolver/CMakeLists.txt @@ -0,0 +1,14 @@ +set(PXR_PREFIX pxr/usd/plugin) +set(PXR_PACKAGE httpResolver) + +pxr_library(${PXR_PACKAGE} + LIBRARIES + ar + arch + PUBLIC_CLASSES + resolver + CPPFILES + resolver.cpp + RESOURCE_FILES + plugInfo.json +) \ No newline at end of file diff --git a/pxr/usdImaging/hdEmscripten/httpResolver/plugInfo.json b/pxr/usdImaging/hdEmscripten/httpResolver/plugInfo.json new file mode 100644 index 0000000000..db660b55cf --- /dev/null +++ b/pxr/usdImaging/hdEmscripten/httpResolver/plugInfo.json @@ -0,0 +1,18 @@ +{ + "Plugins": [ + { + "Info": { + "Types": { + "HttpResolver": { + "bases": ["ArDefaultResolver"] + } + } + }, + "Name": "HttpResolver", + "LibraryPath": "@PLUG_INFO_LIBRARY_PATH@", + "ResourcePath": "@PLUG_INFO_RESOURCE_PATH@", + "Root": "@PLUG_INFO_ROOT@", + "Type": "library" + } + ] +} \ No newline at end of file diff --git a/pxr/usdImaging/hdEmscripten/httpResolver/resolver.cpp b/pxr/usdImaging/hdEmscripten/httpResolver/resolver.cpp new file mode 100644 index 0000000000..08e2fd8266 --- /dev/null +++ b/pxr/usdImaging/hdEmscripten/httpResolver/resolver.cpp @@ -0,0 +1,225 @@ +// IMPORT THIRD-PARTY LIBRARIES +#include +#include +#include +#include +#include + +// IMPORT LOCAL LIBRARIES +#include "resolver.h" + +PXR_NAMESPACE_OPEN_SCOPE + +AR_DEFINE_RESOLVER(HttpResolver, ArDefaultResolver); + +HttpResolver::HttpResolver() : ArDefaultResolver() {} +HttpResolver::~HttpResolver() {} + +struct AssetData { + int ptrToContent; + int length; // Use int for compatibility with JavaScript's setValue; adjust if necessary +}; + +EM_ASYNC_JS(void, fetch_asset, (const char* route, int dataPtr), { + const routeString = UTF8ToString(route); + const absoluteUrl = new URL(routeString); + try { + const response = await fetch(absoluteUrl); + if (!response.ok) throw new Error('Fetch failed: ' + response.statusText); + const buffer = await response.arrayBuffer(); + const length = buffer.byteLength; + const ptr = _malloc(length); + HEAPU8.set(new Uint8Array(buffer), ptr); + + // Correctly set the pointer and length in the AssetData structure + // Note: Assumes dataPtr is a pointer to the structure where the first member is an int pointer + // to the content, and the second is an int for the length. + Module.HEAP32[dataPtr >> 2] = ptr; // Set the pointer + Module.HEAP32[(dataPtr >> 2) + 1] = length; // Set the length + } catch (err) { + console.error("Error in fetch_asset: ", err); + Module.HEAP32[dataPtr >> 2] = 0; // Indicate failure with null pointer + Module.HEAP32[(dataPtr >> 2) + 1] = 0; // and zero length + } +}); + +EM_JS(void, addToLoadedFiles, (const char* path), { + if (typeof loadedFiles === 'undefined'){ + var loadedFiles = []; + } + loadedFiles.push(UTF8ToString(path)); +}); + +std::filesystem::path HttpResolver::FetchAndSaveAsset(const std::string& route, + const std::string& filePath) const { + try { + std::filesystem::path dirPath = std::filesystem::path(filePath).parent_path(); + + // Attempt to create the directory (and any necessary parent directories) + if (std::filesystem::create_directories(dirPath)) { + if (verbose){ + std::cout << "Directories created successfully: " << dirPath << std::endl; + } + } else { + if (verbose){ + std::cout << "Directories already exist or cannot be created.\n"; + } + } + + AssetData* data = new AssetData(); + fetch_asset(route.c_str(), reinterpret_cast(data)); + char *assetContentCString = reinterpret_cast(data->ptrToContent); + saveBinaryAssetContentToFile(assetContentCString, data->length, filePath); + + free(reinterpret_cast(data->ptrToContent)); + delete data; + + } + catch (const std::exception& e){ + std::cout << "Error: " << e.what() << std::endl; + return filePath; + } + + addToLoadedFiles(filePath.c_str()); + return filePath; +} + +void HttpResolver::saveBinaryAssetContentToFile(const char* assetContent, size_t length, const std::string& filePath) const { + + std::ofstream outFile(filePath, std::ios::out | std::ios::binary); + if (outFile) { + // Write the binary content directly to the file + outFile.write(assetContent, length); + outFile.close(); + if (verbose) { + std::cout << "File written successfully." << std::endl; + } + } else { + if (verbose) { + std::cout << "Failed to open file for writing." << std::endl; + } + } +} + +void HttpResolver::setBaseUrl(const std::string& url) const { + baseUrl = url; +} + +void HttpResolver::setBaseTempDir(const std::string& tempDir) const { + baseTempDir = tempDir; +} + +std::string combineUrl(const std::string& baseUrl, const std::string& relativePath) { + // Step 1: Strip off the scheme + auto schemeEnd = baseUrl.find(":/"); + if (schemeEnd == std::string::npos) { + return baseUrl + relativePath; + } + std::string scheme = baseUrl.substr(0, schemeEnd + 3); // Include "://" + std::string basePath = baseUrl.substr(schemeEnd + 3); + + // Extract the domain + auto pathStart = basePath.find('/'); + std::string domain = basePath.substr(0, pathStart); + std::string pathOnly = basePath.substr(pathStart); // Path without the domain + + // Step 2: Use filesystem::path for manipulation + std::filesystem::path pathObj = pathOnly; + pathObj = pathObj.remove_filename(); // Ensure we're manipulating the directory part + pathObj /= relativePath; // Append the relative path + pathObj = pathObj.lexically_normal(); // Normalize the path (resolve "..", ".", etc.) + + // Step 3: Recombine + std::string combinedUrl = scheme + domain + pathObj.string(); + + return combinedUrl; +} + +ArResolvedPath HttpResolver::_Resolve(const std::string& assetPath) const { + if (verbose){ + std::cout << "_Resolve: " << assetPath << std::endl; + } + std::string stringAssetPathCopy = assetPath; + std::filesystem::path savedAssetFilePath = assetPath; + if (std::filesystem::exists(assetPath)){ + if (verbose) { + std::cout << "Already Exists: " << assetPath << std::endl; + } + } + else if (assetPath.find("http") != std::string::npos){ + std::string githubName = "github.com"; + std::string rawGithubName = "raw.githubusercontent.com"; + std::string blob = "/blob"; + + size_t pos = stringAssetPathCopy.find(githubName); + if (pos!= std::string::npos) { + stringAssetPathCopy.replace(pos, githubName.length(), rawGithubName); + } + + size_t pos_blob = stringAssetPathCopy.find(blob); + if (pos_blob!= std::string::npos) { + stringAssetPathCopy.erase(pos_blob, blob.length()); + } + + std::filesystem::path fullHttpRouteAsPath = stringAssetPathCopy; + std::filesystem::path rootHttpRouteAsPath = fullHttpRouteAsPath.parent_path(); + + auto finalBaseUrl = rootHttpRouteAsPath.generic_string() + "/"; + if (verbose){ + std::cout << "http PATH: " << stringAssetPathCopy << std::endl; + std::cout << "finalBaseUrl: " << finalBaseUrl << std::endl; + } + + setBaseUrl(finalBaseUrl); + + std::filesystem::path tempDir = std::filesystem::temp_directory_path(); + // This path is chosen because if an asset is found with the path /../../../ it will go up the tmp directory structure + // in the case of using /tmp/ then all relative paths greater than depth 1, will look the same. using 6 here is arbitrary, + // is there a way to make this always work? + setBaseTempDir(tempDir.generic_string() + "/1/1/1/1/1/1/"); + auto filePath = baseTempDir + fullHttpRouteAsPath.filename().generic_string(); + savedAssetFilePath = FetchAndSaveAsset(assetPath, + filePath); + } + else if (!baseUrl.empty()){ + std::filesystem::path systemPath = stringAssetPathCopy; + std::filesystem::path relativePath = std::filesystem::relative(systemPath, baseTempDir); + + std::string route = combineUrl(baseUrl, relativePath); + if (verbose){ + std::cout << "Relative Path before: " << relativePath << std::endl; + } + + savedAssetFilePath = FetchAndSaveAsset(route, systemPath); + if (verbose){ + std::cout << "Assumed to exist now, trying from baseUrl: " << systemPath << std::endl; + } + } + else { + return ArDefaultResolver::_Resolve(assetPath); + } + + if (verbose){ + std::cout << "ENDDD_Resolve: " << savedAssetFilePath << std::endl; + } + + return ArResolvedPath(savedAssetFilePath); +} + +std::shared_ptr HttpResolver::_OpenAsset(const ArResolvedPath &resolvedPath) const { + if (verbose){ + std::cout << "_OpenAsset: " << resolvedPath.GetPathString() << std::endl; + } + + return ArDefaultResolver::_OpenAsset(resolvedPath); +} + +ArResolvedPath HttpResolver::_ResolveForNewAsset(const std::string &assetPath) const { + if (verbose){ + std::cout << "Resolve for new asset" << std::endl; + } + + return ArDefaultResolver::_ResolveForNewAsset(assetPath); +} + +PXR_NAMESPACE_CLOSE_SCOPE diff --git a/pxr/usdImaging/hdEmscripten/httpResolver/resolver.h b/pxr/usdImaging/hdEmscripten/httpResolver/resolver.h new file mode 100644 index 0000000000..54ed26b6b8 --- /dev/null +++ b/pxr/usdImaging/hdEmscripten/httpResolver/resolver.h @@ -0,0 +1,29 @@ +#pragma once + +// IMPORT THIRD-PARTY LIBRARIES +#include +#include +#include +#include + +PXR_NAMESPACE_OPEN_SCOPE + +class HttpResolver : public ArDefaultResolver { +public: + HttpResolver(); + ~HttpResolver(); + + ArResolvedPath _Resolve(const std::string& path) const override; + std::shared_ptr _OpenAsset(const ArResolvedPath &resolvedPath) const override; + ArResolvedPath _ResolveForNewAsset(const std::string &assetPath) const override; + std::filesystem::path FetchAndSaveAsset(const std::string& baseUrl, const std::string& filePath) const; + void saveBinaryAssetContentToFile(const char* assetContent, size_t length, const std::string& filePath) const; + void setBaseUrl(const std::string &url) const; + void setBaseTempDir(const std::string &tempDir) const; +private: + mutable std::string baseUrl; + bool verbose = false; + mutable std::string baseTempDir; +}; + +PXR_NAMESPACE_CLOSE_SCOPE