From 19bc37cd6f36d3a9ba1e4db1a2e5bf3b97d9559b Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Wed, 30 Oct 2024 12:00:08 -0400 Subject: [PATCH 1/5] add temp warning when disabling peer transfers --- taskvine/src/manager/vine_manager.c | 1 + 1 file changed, 1 insertion(+) diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 65c61d9660..758ca86823 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -3992,6 +3992,7 @@ int vine_enable_peer_transfers(struct vine_manager *q) int vine_disable_peer_transfers(struct vine_manager *q) { debug(D_VINE, "Peer Transfers disabled"); + fprintf(stderr, "warning: Peer Transfers disabled. Using temporary files will force dependent tasks to run on a single worker"); q->peer_transfers_enabled = 0; return 1; } From 2d819866b28f4ed5e820c39ac8e4e98b20d70d97 Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Fri, 1 Nov 2024 10:58:01 -0400 Subject: [PATCH 2/5] without peer transfers, create normal file instead of temps --- taskvine/src/manager/vine_file.c | 8 ++++++++ taskvine/src/manager/vine_file.h | 1 + taskvine/src/manager/vine_manager.c | 11 ++++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/taskvine/src/manager/vine_file.c b/taskvine/src/manager/vine_file.c index d2c236f930..c19fe4f237 100644 --- a/taskvine/src/manager/vine_file.c +++ b/taskvine/src/manager/vine_file.c @@ -245,6 +245,14 @@ struct vine_file *vine_file_temp() return vine_file_create("temp", 0, 0, 0, VINE_TEMP, 0, cache, 0); } +struct vine_file *vine_file_pseudo_temp() +{ + // temp files are always cached at workers until explicitely removed. + vine_cache_level_t cache = VINE_CACHE_LEVEL_WORKFLOW; + + return vine_file_create("temp", 0, 0, 0, VINE_FILE, 0, cache, 0); +} + struct vine_file *vine_file_buffer(const char *data, size_t size, vine_cache_level_t cache, vine_file_flags_t flags) { return vine_file_create("buffer", 0, data, size, VINE_BUFFER, 0, cache, flags); diff --git a/taskvine/src/manager/vine_file.h b/taskvine/src/manager/vine_file.h index 10bb785776..a7e4670c71 100644 --- a/taskvine/src/manager/vine_file.h +++ b/taskvine/src/manager/vine_file.h @@ -62,6 +62,7 @@ char * vine_file_make_file_url( const char * source); struct vine_file *vine_file_local( const char *source, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_url( const char *source, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_temp(); +struct vine_file *vine_file_pseudo_temp(); struct vine_file *vine_file_buffer( const char *buffer, size_t size, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_mini_task( struct vine_task *t, const char *name, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_untar( struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags ); diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 758ca86823..93e54f931f 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -3992,7 +3992,7 @@ int vine_enable_peer_transfers(struct vine_manager *q) int vine_disable_peer_transfers(struct vine_manager *q) { debug(D_VINE, "Peer Transfers disabled"); - fprintf(stderr, "warning: Peer Transfers disabled. Using temporary files will force dependent tasks to run on a single worker"); + fprintf(stderr, "warning: Peer Transfers disabled. Temporary files will be returned to the manager upon creation."); q->peer_transfers_enabled = 0; return 1; } @@ -6177,8 +6177,13 @@ struct vine_file *vine_declare_url(struct vine_manager *m, const char *source, v struct vine_file *vine_declare_temp(struct vine_manager *m) { - struct vine_file *f = vine_file_temp(); - return vine_manager_declare_file(m, f); + if (m->peer_transfers_enabled) { + struct vine_file *f = vine_file_temp(); + return vine_manager_declare_file(m, f); + } else { + struct vine_file *f = vine_file_pseudo_temp(); + return vine_manager_declare_file(m, f); + } } struct vine_file *vine_declare_buffer(struct vine_manager *m, const char *buffer, size_t size, vine_cache_level_t cache, vine_file_flags_t flags) From 065349f2313054d443d89759af305bc0b6b4c9f3 Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Fri, 1 Nov 2024 11:10:54 -0400 Subject: [PATCH 3/5] unique name for local temps --- taskvine/src/manager/vine_file.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/taskvine/src/manager/vine_file.c b/taskvine/src/manager/vine_file.c index c19fe4f237..41f51c5784 100644 --- a/taskvine/src/manager/vine_file.c +++ b/taskvine/src/manager/vine_file.c @@ -249,8 +249,11 @@ struct vine_file *vine_file_pseudo_temp() { // temp files are always cached at workers until explicitely removed. vine_cache_level_t cache = VINE_CACHE_LEVEL_WORKFLOW; + uid_t uid = getuid(); - return vine_file_create("temp", 0, 0, 0, VINE_FILE, 0, cache, 0); + char *name = string_format("temp-local-%u", uid); + return vine_file_create(name, 0, 0, 0, VINE_FILE, 0, cache, 0); + free(name); } struct vine_file *vine_file_buffer(const char *data, size_t size, vine_cache_level_t cache, vine_file_flags_t flags) From 4e31d4b227483d3d90ff953156f4455bb4d55a92 Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Fri, 1 Nov 2024 11:44:10 -0400 Subject: [PATCH 4/5] unlink local temp when done --- taskvine/src/manager/vine_file.c | 4 ++-- taskvine/src/manager/vine_file.h | 2 +- taskvine/src/manager/vine_manager.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/taskvine/src/manager/vine_file.c b/taskvine/src/manager/vine_file.c index 41f51c5784..e0279a9617 100644 --- a/taskvine/src/manager/vine_file.c +++ b/taskvine/src/manager/vine_file.c @@ -245,14 +245,14 @@ struct vine_file *vine_file_temp() return vine_file_create("temp", 0, 0, 0, VINE_TEMP, 0, cache, 0); } -struct vine_file *vine_file_pseudo_temp() +struct vine_file *vine_file_temp_no_peers() { // temp files are always cached at workers until explicitely removed. vine_cache_level_t cache = VINE_CACHE_LEVEL_WORKFLOW; uid_t uid = getuid(); char *name = string_format("temp-local-%u", uid); - return vine_file_create(name, 0, 0, 0, VINE_FILE, 0, cache, 0); + return vine_file_create(name, 0, 0, 0, VINE_FILE, 0, cache, VINE_UNLINK_WHEN_DONE); free(name); } diff --git a/taskvine/src/manager/vine_file.h b/taskvine/src/manager/vine_file.h index a7e4670c71..a358a0b6e4 100644 --- a/taskvine/src/manager/vine_file.h +++ b/taskvine/src/manager/vine_file.h @@ -62,7 +62,7 @@ char * vine_file_make_file_url( const char * source); struct vine_file *vine_file_local( const char *source, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_url( const char *source, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_temp(); -struct vine_file *vine_file_pseudo_temp(); +struct vine_file *vine_file_temp_no_peers(); struct vine_file *vine_file_buffer( const char *buffer, size_t size, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_mini_task( struct vine_task *t, const char *name, vine_cache_level_t cache, vine_file_flags_t flags ); struct vine_file *vine_file_untar( struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags ); diff --git a/taskvine/src/manager/vine_manager.c b/taskvine/src/manager/vine_manager.c index 93e54f931f..631eff4646 100644 --- a/taskvine/src/manager/vine_manager.c +++ b/taskvine/src/manager/vine_manager.c @@ -6181,7 +6181,7 @@ struct vine_file *vine_declare_temp(struct vine_manager *m) struct vine_file *f = vine_file_temp(); return vine_manager_declare_file(m, f); } else { - struct vine_file *f = vine_file_pseudo_temp(); + struct vine_file *f = vine_file_temp_no_peers(); return vine_manager_declare_file(m, f); } } From 262fb70d8ff41cea1fdbbe71b325f3a76938eb9e Mon Sep 17 00:00:00 2001 From: Colin Thomas Date: Tue, 5 Nov 2024 09:51:43 -0500 Subject: [PATCH 5/5] correct uuid generation for temp names --- taskvine/src/manager/vine_file.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/taskvine/src/manager/vine_file.c b/taskvine/src/manager/vine_file.c index e0279a9617..c11be4355b 100644 --- a/taskvine/src/manager/vine_file.c +++ b/taskvine/src/manager/vine_file.c @@ -15,6 +15,7 @@ See the file COPYING for details. #include "stringtools.h" #include "timestamp.h" #include "unlink_recursive.h" +#include "uuid.h" #include "xxmalloc.h" #include @@ -249,9 +250,10 @@ struct vine_file *vine_file_temp_no_peers() { // temp files are always cached at workers until explicitely removed. vine_cache_level_t cache = VINE_CACHE_LEVEL_WORKFLOW; - uid_t uid = getuid(); + cctools_uuid_t uuid; + cctools_uuid_create(&uuid); - char *name = string_format("temp-local-%u", uid); + char *name = string_format("temp-local-%s", uuid.str); return vine_file_create(name, 0, 0, 0, VINE_FILE, 0, cache, VINE_UNLINK_WHEN_DONE); free(name); }