diff --git a/pyproject.toml b/pyproject.toml index d676575..5e1aea4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "zfslib" -version = "0.9.1" +version = "0.9.3" description = "ZFS Utilities For Python3" license = "MIT" authors = ["Timothy C. Quinn"] diff --git a/src/zfslib/zfslib.py b/src/zfslib/zfslib.py index 1840a9d..c2846f9 100644 --- a/src/zfslib/zfslib.py +++ b/src/zfslib/zfslib.py @@ -361,9 +361,9 @@ def get_dataset(self, name): if isinstance(ds, Dataset): return ds if isinstance(self, Pool): - raise KeyError("Dataset '{}' not found in Pool '{}'.".format(name, self.name)) + raise KeyError(f"Dataset '{name}' not found in Pool '{self.name}'.") else: - raise KeyError("Dataset '{}' not found in Dataset '{}'.".format(name, self.path)) + raise KeyError(f"Dataset '{name}' not found in Dataset '{self.path}'.") # returns list(of str) or if with_depth == True then list(of tuple(of depth, Dataset)) @@ -380,8 +380,8 @@ def get_all_datasets(self, with_depth=False, depth=0): # WARNING - Using this function to filter if snapshot contains a folder def get_snapshots(self, flt=True, index=False): if flt is True: flt = lambda _:True - assert inspect.isfunction(flt), "flt must either be True or a Function. Got: {}".format(type(flt)) - assert isinstance(index, bool), "index must be a boolean. Got: {}".format(type(index)) + assert inspect.isfunction(flt), f"flt must either be True or a Function. Got: {type(flt)}" + assert isinstance(index, bool), f"index must be a boolean. Got: {type(index)}" _ds_path = self.path res = [] for idx, c in enumerate(self.children): @@ -423,7 +423,7 @@ def __assert(k, types, default=None, to_datetime=False): else: if not k in find_opts: return default v = find_opts[k] - assert isinstance(v, types), 'Invalid type for param {}. Expecting {} but got: {}'.format(k, types, type(v)) + assert isinstance(v, types), f'Invalid type for param {k}. Expecting {types} but got: {type(v)}' if to_datetime and not isinstance(v, datetime): return datetime(v.year, v.month, v.day) @@ -476,7 +476,7 @@ def __fil_dt(snap): if not tdelta is None: raise AssertionError("tdelta cannot be specified when both dt_from and dt_to are specified") if dt_from >= dt_to: - raise AssertionError("dt_from ({}) must be < dt_to ({})".format(dt_from, dt_to)) + raise AssertionError(f"dt_from ({dt_from}) must be < dt_to ({dt_to})") (dt_f, dt_t) = (dt_from, dt_to) f=__fil_dt @@ -558,7 +558,7 @@ def __tv(k, v): if v is None: return None if isinstance(v, str): return [v] if isinstance(v, list): return v - raise AssertionError("{} can only be a str or list. Got: {}".format(k, type(v))) + raise AssertionError(f"{k} can only be a str or list. Got: {type(v)}") file_type = __tv('file_type', file_type) @@ -591,8 +591,14 @@ def __row(s): rows = list(map(lambda s: __row(s), stdout.splitlines())) diffs = [] - for row in rows: + for i, row in enumerate(rows): + # if i == 429: + # print("HERE") d = Diff(row, snap_left, snap_right) + if d.path_full.find('(on_delete_queue)') > 0: + # It looks to be an artefact of ZFS that does not actually exist in FS + # https://github.com/openzfs/zfs/blob/master/lib/libzfs/libzfs_diff.c + continue if not file_type is None and not d.file_type in file_type: continue if not chg_type is None and not d.chg_type in chg_type: continue @@ -614,6 +620,7 @@ def __row(s): bIgn = True break if bIgn: continue + diffs.append(d) return diffs @@ -641,12 +648,12 @@ def _get_mounted(self): # path must be an actual path on the system being analyzed def get_rel_path(self, path): self.assertHaveMounts() - assert isinstance(path, str), "argument passed is not a string. Got: {}".format(type(path)) + assert isinstance(path, str), f"argument passed is not a string. Got: {type(path)}" p_real = os.path.abspath( expand_user(path) ) p_real = os.path.realpath(p_real) mp = self.mountpoint if not p_real.find(mp) == 0: - raise KeyError('path given is not in current dataset mountpoint {}. Path: {}'.format(mp, path)) + raise KeyError(f'path given is not in current dataset mountpoint {mp}. Path: {path}') return p_real.replace(mp, '') @@ -690,7 +697,7 @@ def _get_snap_path(self): assert isinstance(self.parent, Dataset), \ "This function is only available for Snapshots of Datasets not Pools" self.parent.assertHaveMounts() - return "{}/.zfs/snapshot/{}".format(self.parent.mountpoint, self.name) + return f"{self.parent.mountpoint}/.zfs/snapshot/{self.name}" snap_path = property(_get_snap_path) @@ -706,7 +713,7 @@ def resolve_snap_path(self, path): "This function is only available for Snapshots of Datasets not Pools" self.parent.assertHaveMounts() assert self.parent.mounted, \ - "Parent Dataset {} is not mounted. Please verify datsset.mounted before calling this function".format(self.parent) + f"Parent Dataset {self.parent} is not mounted. Please verify datsset.mounted before calling this function" if path is None or not isinstance(path, str) or path.strip() == '': assert 0, "path must be a non-blank string" @@ -715,7 +722,7 @@ def resolve_snap_path(self, path): snap_path_base = self.snap_path ds_mp = self.dataset.mountpoint if path_neweal.find(ds_mp) == -1: - raise KeyError("Path given is not within the dataset's mountpoint of {}. Path passed: {}".format(ds_mp, path)) + raise KeyError(f"Path given is not within the dataset's mountpoint of {ds_mp}. Path passed: {path}") snap_path = "{}{}".format(snap_path_base, path_neweal.replace(ds_mp, '')) if os.path.exists(snap_path): return (True, snap_path) @@ -762,17 +769,17 @@ def __init__(self, row, snap_left, snap_right): self.no_from_snap=True snap_left = None elif not isinstance(snap_left, Snapshot): - raise AssertionError("snap_left must be either a Snapshot or str('na-first'). Got: {}".format(type(snap_left))) + raise AssertionError(f"snap_left must be either a Snapshot or str('na-first'). Got: {type(snap_left)}") if isinstance(snap_right, str) and snap_right == '(present)': self.to_present=True snap_right = None elif not isinstance(snap_right, Snapshot): - raise AssertionError("snap_left must be either a Snapshot. Got: {}".format(type(snap_right))) + raise AssertionError(f"snap_left must be either a Snapshot. Got: {type(snap_right)}") - if not self.no_from_snap and not self.to_present and snap_left.creation >= snap_right.creation: - raise AssertionError("diff from creation ({}) is > or = to diff_to creation ({})".format(snap_left.creation, snap_right.creation)) + if not self.no_from_snap and not self.to_present and snap_left.creation > snap_right.creation: + raise AssertionError(f"diff from creation ({snap_left.creation}) is > to diff_to creation ({snap_right.creation})") self.snap_left = snap_left self.snap_right = snap_right @@ -783,7 +790,7 @@ def __init__(self, row, snap_left, snap_right): elif len(row) == 5: (inode_ts, chg_type, file_type, path, path_new) = row else: - raise Exception("Unexpected len: {}. Row = {}".format(len(row), row)) + raise Exception(f"Unexpected len: {len(row)}. Row = {row}") chg_time = datetime.fromtimestamp(int(inode_ts[:inode_ts.find('.')])) self.chg_ts = inode_ts @@ -834,13 +841,13 @@ def _get_snap_path_right(self): @staticmethod def get_file_type(s): assert (isinstance(s, str) and not s == ''), "argument must be a non-empty string" - assert s in Diff.FILE_TYPES, "ZFS Diff File type is invalid: '{}'".format(s) + assert s in Diff.FILE_TYPES, f"ZFS Diff File type is invalid: '{s}'" return Diff.FILE_TYPES[s] @staticmethod def get_change_type(s): assert (isinstance(s, str) and not s == ''), "argument must be a non-empty string" - assert s in Diff.CHANGE_TYPES, "ZFS Diff Change type is invalid: '{}'".format(s) + assert s in Diff.CHANGE_TYPES, f"ZFS Diff Change type is invalid: '{s}'" return Diff.CHANGE_TYPES[s] def __str__(self): @@ -859,57 +866,57 @@ def __str__(self): # buildTimedelta() # Builds timedelta from string: # . tdelta is a timedelta -or- str(nC) where: n is an integer > 0 and C is one of: -# . y=year, m=month, w=week, d=day, H=hour, M=minute, s=second +# . Y=year, M=month, W=week, D=day, h=hour, m=minute, s=second # Note: month and year are imprecise and assume 30.4 and 365 days -def buildTimedelta(tdelta): +def buildTimedelta(tdelta) -> timedelta: if isinstance(tdelta, timedelta): return tdelta if not isinstance(tdelta, str): - raise AssertionError('tdelta must be a string') + raise KeyError('tdelta must be a string') elif len(tdelta) < 2: - raise AssertionError('len(tdelta) must be >= 2') + raise KeyError('len(tdelta) must be >= 2') n = tdelta[:-1] try: n = int(n) - if n < 1: raise AssertionError('tdelta must be > 0') + if n < 1: raise KeyError('tdelta must be > 0') except ValueError as ex: - raise AssertionError('Value passed for tdelta does not contain a number: {}'.format(tdelta)) + raise KeyError(f'Value passed for tdelta does not contain a number: {tdelta}') c = tdelta[-1:] - if c == 'H': + if c == 'h': return timedelta(hours=n) - elif c == 'M': + elif c == 'm': return timedelta(minutes=n) - elif c == 'S': + elif c == 's': return timedelta(seconds=n) - elif c == 'd': + elif c == 'D': return timedelta(days=n) - elif c == 'm': - return timedelta(days=n*(365/12)) - elif c == 'w': + elif c == 'W': return timedelta(weeks=n) - elif c == 'y': + elif c == 'M': + return timedelta(days=n*(365/12)) + elif c == 'Y': return timedelta(days=n*365) else: - raise AssertionError('Unexpected datetime identifier, expecting one of y,m,w,d,H,M,S.') + raise KeyError('Unexpected datetime identifier, expecting one of Y,M,W,D,h,m,s') # calcDateRange() # Calculates a date range based on tdelta string passed # tdelta is a timedelta -or- str(nC) where: n is an integer > 0 and C is one of: -# . y=year, m=month, w=week, d=day, H=hour, M=minute, s=second +# . Y=year, M=month, W=week, D=day, h=hour, m=minute, s=second # If dt_from is defined, return tuple: (dt_from, dt_from+tdelta) # If dt_to is defined, return tuple: (dt_from-tdelta, dt_to) -def calcDateRange(tdelta, dt_from=None, dt_to=None): - if tdelta is None: raise AssertionError('tdelta is required') +def calcDateRange(tdelta, dt_from:datetime=None, dt_to:datetime=None) -> tuple: + if tdelta is None: raise KeyError('tdelta is required') if dt_from and dt_to: - raise AssertionError('Only one of dt_from or dt_to must be defined') + raise KeyError('Only one of dt_from or dt_to must be defined') elif (not dt_from and not dt_to): - raise AssertionError('Please specify one of dt_from or dt_to') + raise KeyError('Please specify one of dt_from or dt_to') elif dt_from and not isinstance(dt_from, datetime): - raise AssertionError('dt_from must be a datetime') + raise KeyError('dt_from must be a datetime') elif dt_to and not isinstance(dt_to, datetime): - raise AssertionError('dt_to must be a datetime') + raise KeyError('dt_to must be a datetime') td = buildTimedelta(tdelta) @@ -920,7 +927,7 @@ def calcDateRange(tdelta, dt_from=None, dt_to=None): def splitPath(s): - assert isinstance(s, str), "String not passed. Got: {}".format(type(s)) + assert isinstance(s, str), f"String not passed. Got: {type(s)}" s = s.strip() assert not s == '', "Empty string passed" f = os.path.basename(s) @@ -976,7 +983,25 @@ def expand_user(path): return pathlib.Path(path).expanduser() +# Ignore snapshots with exact same timestamp +# . Edge cases that can happen and muck up stuff +# . Handles list(of Snapshot) and list(of tuple(of idx, Snapshot)) +def removeDuplicateSnapshotsByDate(snapshots): + _ret = [] + for i, snap_rec in enumerate(snapshots): + if isinstance(snap_rec, Snapshot): + snap = snap_rec + elif isinstance(snap_rec, tuple): + (idx, snap) = snap_rec + else: + raise Exception(f"Invalid snapshot list passed. Got {type(snap_rec)} at record {i}") + + if i > 0 and snap.creation == snap_last.creation: continue + + _ret.append(snap_rec) + snap_last = snap + return _ret ''' END Utilities ''' diff --git a/tests/test_zfslib.py b/tests/test_zfslib.py index a1cf7bb..50a6de5 100644 --- a/tests/test_zfslib.py +++ b/tests/test_zfslib.py @@ -337,7 +337,7 @@ def test_snapable_find_snapshots(self): snaps = ds.find_snapshots({'name': '*zsys_*e*', 'tdelta': timedelta(hours=36) ,'dt_to': dt_from_creation('1609362247')}) self.assertEqual(len(snaps), 5) - snaps2 = ds.find_snapshots({'name': '*zsys_*e*', 'tdelta': '36H' + snaps2 = ds.find_snapshots({'name': '*zsys_*e*', 'tdelta': '36h' ,'dt_to': dt_from_creation('1609362247')}) self.assertEqual(len(snaps2), len(snaps)) for i, snap in enumerate(snaps): @@ -348,7 +348,7 @@ def test_snapable_find_snapshots(self): ,'dt_from': dt_from_creation('1608233673'), 'tdelta': timedelta(hours=48)}) self.assertEqual(len(snaps), 5) snaps2 = ds.find_snapshots({'name': '*zsys_*w*' - ,'dt_from': dt_from_creation('1608233673'), 'tdelta': '48H'}) + ,'dt_from': dt_from_creation('1608233673'), 'tdelta': '48h'}) self.assertEqual(len(snaps2), len(snaps)) for i, snap in enumerate(snaps): self.assertIs(snap, snaps2[i]) @@ -384,7 +384,7 @@ def test_snapable_find_snapshots(self): with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'dt_from': 'asdf'}) with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'dt_to': 'asdf'}) with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'tdelta': 10}) - with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'tdelta': '-10H'}) + with self.assertRaises(KeyError): snaps = ds.find_snapshots({'tdelta': '-10h'}) with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'index': 1}) with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'dt_to': dt_date(2020, 12, 20) @@ -392,7 +392,7 @@ def test_snapable_find_snapshots(self): with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'dt_to': dt_date(2020, 12, 21) ,'dt_from': dt_date(2020, 12, 20) - ,'tdelta': "1H"}) + ,'tdelta': "1h"}) # tested against data_nomounts.tsv @@ -422,45 +422,45 @@ def test_no_mounts(self): # Test General Utilities def test_buildTimedelta(self): self.assertEqual(zfs.buildTimedelta(timedelta(seconds=10)), timedelta(seconds=10)) - self.assertEqual(zfs.buildTimedelta('1y'), timedelta(days=365)) - self.assertEqual(zfs.buildTimedelta('1m'), timedelta(days=(365/12))) - self.assertEqual(zfs.buildTimedelta('1w'), timedelta(weeks=1)) - self.assertEqual(zfs.buildTimedelta('10d'), timedelta(days=10)) - self.assertEqual(zfs.buildTimedelta('10H'), timedelta(hours=10)) - self.assertEqual(zfs.buildTimedelta('10M'), timedelta(minutes=10)) - self.assertEqual(zfs.buildTimedelta('10S'), timedelta(seconds=10)) + self.assertEqual(zfs.buildTimedelta('1Y'), timedelta(days=365)) + self.assertEqual(zfs.buildTimedelta('1M'), timedelta(days=(365/12))) + self.assertEqual(zfs.buildTimedelta('1W'), timedelta(weeks=1)) + self.assertEqual(zfs.buildTimedelta('10D'), timedelta(days=10)) + self.assertEqual(zfs.buildTimedelta('10h'), timedelta(hours=10)) + self.assertEqual(zfs.buildTimedelta('10m'), timedelta(minutes=10)) + self.assertEqual(zfs.buildTimedelta('10s'), timedelta(seconds=10)) # Negative tests with self.assertRaises(TypeError): zfs.buildTimedelta() - with self.assertRaises(TypeError): zfs.buildTimedelta('1H', True) - with self.assertRaises(AssertionError): zfs.buildTimedelta(None) - with self.assertRaises(AssertionError): zfs.buildTimedelta(datetime.now()) - with self.assertRaises(AssertionError): zfs.buildTimedelta('1') - with self.assertRaises(AssertionError): zfs.buildTimedelta('aH') - with self.assertRaises(AssertionError): zfs.buildTimedelta('-1H') - with self.assertRaises(AssertionError): zfs.buildTimedelta('1X') + with self.assertRaises(TypeError): zfs.buildTimedelta('1h', True) + with self.assertRaises(KeyError): zfs.buildTimedelta(None) + with self.assertRaises(KeyError): zfs.buildTimedelta(datetime.now()) + with self.assertRaises(KeyError): zfs.buildTimedelta('1') + with self.assertRaises(KeyError): zfs.buildTimedelta('ah') + with self.assertRaises(KeyError): zfs.buildTimedelta('-1h') + with self.assertRaises(KeyError): zfs.buildTimedelta('1X') def test_calcDateRange(self): _now = datetime.now() _weekago = datetime.now() - timedelta(weeks=1) - self.assertEqual(zfs.calcDateRange('1d', dt_to=_now), \ + self.assertEqual(zfs.calcDateRange('1D', dt_to=_now), \ (_now - timedelta(days=1), _now) ) - self.assertEqual(zfs.calcDateRange('2d', dt_from=_weekago), \ + self.assertEqual(zfs.calcDateRange('2D', dt_from=_weekago), \ (_weekago, _weekago + timedelta(days=2)) ) - self.assertEqual(zfs.calcDateRange('3m', dt_to=_now), \ + self.assertEqual(zfs.calcDateRange('3M', dt_to=_now), \ (_now - timedelta(days=(3 * (365/12))), _now) ) # Negative tests with self.assertRaises(TypeError): zfs.calcDateRange() with self.assertRaises(TypeError): zfs.calcDateRange(1,2,3,4) - with self.assertRaises(AssertionError): zfs.calcDateRange(None) - with self.assertRaises(AssertionError): zfs.calcDateRange('1H', _now, _now) - with self.assertRaises(AssertionError): zfs.calcDateRange('1H', dt_from=None) - with self.assertRaises(AssertionError): zfs.calcDateRange('1H', dt_to=None) + with self.assertRaises(KeyError): zfs.calcDateRange(None) + with self.assertRaises(KeyError): zfs.calcDateRange('1h', _now, _now) + with self.assertRaises(KeyError): zfs.calcDateRange('1h', dt_from=None) + with self.assertRaises(KeyError): zfs.calcDateRange('1h', dt_to=None) def test_splitPath(self): diff --git a/tests/test_zfslib_online.py b/tests/test_zfslib_online.py index 0ec8459..e73e91d 100644 --- a/tests/test_zfslib_online.py +++ b/tests/test_zfslib_online.py @@ -47,7 +47,7 @@ def _checkPath(path): b_cl = False elif diff.chg_type == 'R': pass - + if b_cl: _checkPath(diff.snap_path_left) if b_cr: _checkPath(diff.snap_path_right) @@ -61,7 +61,7 @@ def test_get_diffs(self): def _verify_diffs(self, step, diffs): self.assertIsInstance(diffs, list, "Step: {}. Object is not a list. Got: {}".format(step, type(diffs))) if len(diffs) == 0: return - for diff in diffs: + for i, diff in enumerate(diffs): try: validate_diff(diff) except Exception as ex: @@ -241,4 +241,4 @@ def test_resolve_snap_path(self): if __name__ == "__main__": unittest.main() - sys.exit(0) \ No newline at end of file + sys.exit(0)