can the error messages be more specific?

[simonm3]

I am trying to list a type of file in an s3 bucket:

res = s3.glob("job-data/*/results/resultsplus.csv")
res2 = list(res)

This fails after 4 minutes (it is a big bucket) with "permission error". However it does not tell me which key it failed on. So firstly this error message should report the key. Secondly any suggestions welcome as to an easy way to find it!

I note also that if any folders in the whole bucket are not defined then glob fails. This is even the case for files outside the hierarchy e.g. if I glob on xxxx and aaaa/bbbb exists but aaaa/ does not then glob fails even though aaaa is not even in the search path. Maybe this is an aws s3 thing?

ClientError Traceback (most recent call last)
~\Anaconda3\lib\site-packages\fs_s3fs_s3fs.py in s3errors(path)
175 try:
--> 176 yield
177 except ClientError as error:

~\Anaconda3\lib\site-packages\fs_s3fs_s3fs.py in _get_object(self, path, key)
352 obj = self.s3.Object(self._bucket_name, _key)
--> 353 obj.load()
354 except errors.ResourceNotFound:

~\Anaconda3\lib\site-packages\boto3\resources\factory.py in do_action(self, *args, **kwargs)
504 def do_action(self, *args, **kwargs):
--> 505 response = action(self, *args, **kwargs)
506 self.meta.data = response

~\Anaconda3\lib\site-packages\boto3\resources\action.py in call(self, parent, *args, **kwargs)
82
---> 83 response = getattr(parent.meta.client, operation_name)(**params)
84

~\Anaconda3\lib\site-packages\botocore\client.py in _api_call(self, *args, **kwargs)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358

~\Anaconda3\lib\site-packages\botocore\client.py in _make_api_call(self, operation_name, api_params)
660 error_class = self.exceptions.from_code(error_code)
--> 661 raise error_class(parsed_response, operation_name)
662 else:

ClientError: An error occurred (404) when calling the HeadObject operation: Not Found

During handling of the above exception, another exception occurred:

ResourceNotFound Traceback (most recent call last)
~\Anaconda3\lib\site-packages\fs_s3fs_s3fs.py in _get_object(self, path, key)
352 obj = self.s3.Object(self._bucket_name, _key)
--> 353 obj.load()
354 except errors.ResourceNotFound:

~\Anaconda3\lib\contextlib.py in exit(self, type, value, traceback)
129 try:
--> 130 self.gen.throw(type, value, traceback)
131 except StopIteration as exc:

~\Anaconda3\lib\site-packages\fs_s3fs_s3fs.py in s3errors(path)
185 if http_status == 404:
--> 186 raise errors.ResourceNotFound(path)
187 elif http_status == 403:

ResourceNotFound: resource '/job-data/job-337/results' not found

During handling of the above exception, another exception occurred:

ClientError Traceback (most recent call last)
~\Anaconda3\lib\site-packages\fs_s3fs_s3fs.py in s3errors(path)
175 try:
--> 176 yield
177 except ClientError as error:

~\Anaconda3\lib\site-packages\fs_s3fs_s3fs.py in _get_object(self, path, key)
358 )
--> 359 obj.load()
360 return obj

~\Anaconda3\lib\site-packages\boto3\resources\factory.py in do_action(self, *args, **kwargs)
504 def do_action(self, *args, **kwargs):
--> 505 response = action(self, *args, **kwargs)
506 self.meta.data = response

~\Anaconda3\lib\site-packages\boto3\resources\action.py in call(self, parent, *args, **kwargs)
82
---> 83 response = getattr(parent.meta.client, operation_name)(**params)
84

~\Anaconda3\lib\site-packages\botocore\client.py in _api_call(self, *args, **kwargs)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358

~\Anaconda3\lib\site-packages\botocore\client.py in _make_api_call(self, operation_name, api_params)
660 error_class = self.exceptions.from_code(error_code)
--> 661 raise error_class(parsed_response, operation_name)
662 else:

ClientError: An error occurred (403) when calling the HeadObject operation: Forbidden

During handling of the above exception, another exception occurred:

PermissionDenied Traceback (most recent call last)
in
1 # with results
2 res = s3.glob("job-data/*/results/resultsplus.csv")
----> 3 res2 = list(res)

~\Anaconda3\lib\site-packages\fs\glob.py in _make_iter(self, search, namespaces)
152 max_depth=None if recursive else levels,
153 search=search,
--> 154 exclude_dirs=self.exclude_dirs,
155 ):
156 if info.is_dir:

~\Anaconda3\lib\site-packages\fs\walk.py in info(self, fs, path, namespaces)
383 """
384 _walk = self._iter_walk(fs, path=path, namespaces=namespaces)
--> 385 for _path, info in _walk:
386 if info is not None:
387 yield join(_path, info.name), info

~\Anaconda3\lib\site-packages\fs\walk.py in _walk_breadth(self, fs, path, namespaces)
403 while queue:
404 dir_path = pop()
--> 405 for info in self._scan(fs, dir_path, namespaces=namespaces):
406 if info.is_dir:
407 _depth = self._calculate_depth(dir_path) - depth + 1

~\Anaconda3\lib\site-packages\fs\walk.py in _scan(self, fs, dir_path, namespaces)
279 except FSError as error:
280 if not self.on_error(dir_path, error):
--> 281 six.reraise(type(error), error)
282
283 def walk(

~\Anaconda3\lib\site-packages\six.py in reraise(tp, value, tb)
690 value = tp()
691 if value.traceback is not tb:
--> 692 raise value.with_traceback(tb)
693 raise value
694 finally:

PermissionDenied: Forbidden

time: 4min 2s