Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

vine: handling temp files when peer transfers disabled #3964

Merged
merged 5 commits into from
Nov 5, 2024
Merged

vine: handling temp files when peer transfers disabled #3964

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
19bc37c
add temp warning when disabling peer transfers
colinthomas-z80 Oct 30, 2024
2d81986
without peer transfers, create normal file instead of temps
colinthomas-z80 Nov 1, 2024
065349f
unique name for local temps
colinthomas-z80 Nov 1, 2024
4e31d4b
unlink local temp when done
colinthomas-z80 Nov 1, 2024
262fb70
correct uuid generation for temp names
colinthomas-z80 Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions taskvine/src/manager/vine_file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,14 @@ struct vine_file *vine_file_temp()
return vine_file_create("temp", 0, 0, 0, VINE_TEMP, 0, cache, 0);
}

struct vine_file *vine_file_pseudo_temp()
{
// temp files are always cached at workers until explicitely removed.
vine_cache_level_t cache = VINE_CACHE_LEVEL_WORKFLOW;

return vine_file_create("temp", 0, 0, 0, VINE_FILE, 0, cache, 0);
dthain marked this conversation as resolved.
Show resolved Hide resolved
}

struct vine_file *vine_file_buffer(const char *data, size_t size, vine_cache_level_t cache, vine_file_flags_t flags)
{
return vine_file_create("buffer", 0, data, size, VINE_BUFFER, 0, cache, flags);
Expand Down
1 change: 1 addition & 0 deletions taskvine/src/manager/vine_file.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ char * vine_file_make_file_url( const char * source);
struct vine_file *vine_file_local( const char *source, vine_cache_level_t cache, vine_file_flags_t flags );
struct vine_file *vine_file_url( const char *source, vine_cache_level_t cache, vine_file_flags_t flags );
struct vine_file *vine_file_temp();
struct vine_file *vine_file_pseudo_temp();
struct vine_file *vine_file_buffer( const char *buffer, size_t size, vine_cache_level_t cache, vine_file_flags_t flags );
struct vine_file *vine_file_mini_task( struct vine_task *t, const char *name, vine_cache_level_t cache, vine_file_flags_t flags );
struct vine_file *vine_file_untar( struct vine_file *f, vine_cache_level_t cache, vine_file_flags_t flags );
Expand Down
10 changes: 8 additions & 2 deletions taskvine/src/manager/vine_manager.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3992,6 +3992,7 @@ int vine_enable_peer_transfers(struct vine_manager *q)
int vine_disable_peer_transfers(struct vine_manager *q)
{
debug(D_VINE, "Peer Transfers disabled");
fprintf(stderr, "warning: Peer Transfers disabled. Temporary files will be returned to the manager upon creation.");
q->peer_transfers_enabled = 0;
return 1;
}
Expand Down Expand Up @@ -6176,8 +6177,13 @@ struct vine_file *vine_declare_url(struct vine_manager *m, const char *source, v

struct vine_file *vine_declare_temp(struct vine_manager *m)
{
struct vine_file *f = vine_file_temp();
return vine_manager_declare_file(m, f);
if (m->peer_transfers_enabled) {
struct vine_file *f = vine_file_temp();
return vine_manager_declare_file(m, f);
} else {
struct vine_file *f = vine_file_pseudo_temp();
return vine_manager_declare_file(m, f);
}
}

struct vine_file *vine_declare_buffer(struct vine_manager *m, const char *buffer, size_t size, vine_cache_level_t cache, vine_file_flags_t flags)
Expand Down
Loading