Skip to content

Commit

Permalink
Alter Time Delta Codes to Match Usage in ISO 8601
Browse files Browse the repository at this point in the history 
- Update TimeDelta code from y to Y (year), m to M (month) etc. to be more intuitive
- Patch bug with bad data showing up in diff output
- Cleanup of print statements
  • Loading branch information
@JavaScriptDude
JavaScriptDude authored Oct 6, 2022
1 parent 5bb37bb commit 6329927
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 72 deletions.
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "zfslib"
version = "0.9.1"
version = "0.9.3"
description = "ZFS Utilities For Python3"
license = "MIT"
authors = ["Timothy C. Quinn"]
Expand Down
111 changes: 68 additions & 43 deletions src/zfslib/zfslib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,9 +361,9 @@ def get_dataset(self, name):
if isinstance(ds, Dataset): return ds

if isinstance(self, Pool):
raise KeyError("Dataset '{}' not found in Pool '{}'.".format(name, self.name))
raise KeyError(f"Dataset '{name}' not found in Pool '{self.name}'.")
else:
raise KeyError("Dataset '{}' not found in Dataset '{}'.".format(name, self.path))
raise KeyError(f"Dataset '{name}' not found in Dataset '{self.path}'.")


# returns list(of str) or if with_depth == True then list(of tuple(of depth, Dataset))
Expand All @@ -380,8 +380,8 @@ def get_all_datasets(self, with_depth=False, depth=0):
# WARNING - Using this function to filter if snapshot contains a folder
def get_snapshots(self, flt=True, index=False):
if flt is True: flt = lambda _:True
assert inspect.isfunction(flt), "flt must either be True or a Function. Got: {}".format(type(flt))
assert isinstance(index, bool), "index must be a boolean. Got: {}".format(type(index))
assert inspect.isfunction(flt), f"flt must either be True or a Function. Got: {type(flt)}"
assert isinstance(index, bool), f"index must be a boolean. Got: {type(index)}"
_ds_path = self.path
res = []
for idx, c in enumerate(self.children):
Expand Down Expand Up @@ -423,7 +423,7 @@ def __assert(k, types, default=None, to_datetime=False):
else:
if not k in find_opts: return default
v = find_opts[k]
assert isinstance(v, types), 'Invalid type for param {}. Expecting {} but got: {}'.format(k, types, type(v))
assert isinstance(v, types), f'Invalid type for param {k}. Expecting {types} but got: {type(v)}'

if to_datetime and not isinstance(v, datetime):
return datetime(v.year, v.month, v.day)
Expand Down Expand Up @@ -476,7 +476,7 @@ def __fil_dt(snap):
if not tdelta is None:
raise AssertionError("tdelta cannot be specified when both dt_from and dt_to are specified")
if dt_from >= dt_to:
raise AssertionError("dt_from ({}) must be < dt_to ({})".format(dt_from, dt_to))
raise AssertionError(f"dt_from ({dt_from}) must be < dt_to ({dt_to})")
(dt_f, dt_t) = (dt_from, dt_to)
f=__fil_dt

Expand Down Expand Up @@ -558,7 +558,7 @@ def __tv(k, v):
if v is None: return None
if isinstance(v, str): return [v]
if isinstance(v, list): return v
raise AssertionError("{} can only be a str or list. Got: {}".format(k, type(v)))
raise AssertionError(f"{k} can only be a str or list. Got: {type(v)}")


file_type = __tv('file_type', file_type)
Expand Down Expand Up @@ -591,8 +591,14 @@ def __row(s):

rows = list(map(lambda s: __row(s), stdout.splitlines()))
diffs = []
for row in rows:
for i, row in enumerate(rows):
# if i == 429:
# print("HERE")
d = Diff(row, snap_left, snap_right)
if d.path_full.find('(on_delete_queue)') > 0:
# It looks to be an artefact of ZFS that does not actually exist in FS
# https://github.com/openzfs/zfs/blob/master/lib/libzfs/libzfs_diff.c
continue
if not file_type is None and not d.file_type in file_type: continue
if not chg_type is None and not d.chg_type in chg_type: continue

Expand All @@ -614,6 +620,7 @@ def __row(s):
bIgn = True
break
if bIgn: continue

diffs.append(d)

return diffs
Expand Down Expand Up @@ -641,12 +648,12 @@ def _get_mounted(self):
# path must be an actual path on the system being analyzed
def get_rel_path(self, path):
self.assertHaveMounts()
assert isinstance(path, str), "argument passed is not a string. Got: {}".format(type(path))
assert isinstance(path, str), f"argument passed is not a string. Got: {type(path)}"
p_real = os.path.abspath( expand_user(path) )
p_real = os.path.realpath(p_real)
mp = self.mountpoint
if not p_real.find(mp) == 0:
raise KeyError('path given is not in current dataset mountpoint {}. Path: {}'.format(mp, path))
raise KeyError(f'path given is not in current dataset mountpoint {mp}. Path: {path}')
return p_real.replace(mp, '')


Expand Down Expand Up @@ -690,7 +697,7 @@ def _get_snap_path(self):
assert isinstance(self.parent, Dataset), \
"This function is only available for Snapshots of Datasets not Pools"
self.parent.assertHaveMounts()
return "{}/.zfs/snapshot/{}".format(self.parent.mountpoint, self.name)
return f"{self.parent.mountpoint}/.zfs/snapshot/{self.name}"
snap_path = property(_get_snap_path)


Expand All @@ -706,7 +713,7 @@ def resolve_snap_path(self, path):
"This function is only available for Snapshots of Datasets not Pools"
self.parent.assertHaveMounts()
assert self.parent.mounted, \
"Parent Dataset {} is not mounted. Please verify datsset.mounted before calling this function".format(self.parent)
f"Parent Dataset {self.parent} is not mounted. Please verify datsset.mounted before calling this function"

if path is None or not isinstance(path, str) or path.strip() == '':
assert 0, "path must be a non-blank string"
Expand All @@ -715,7 +722,7 @@ def resolve_snap_path(self, path):
snap_path_base = self.snap_path
ds_mp = self.dataset.mountpoint
if path_neweal.find(ds_mp) == -1:
raise KeyError("Path given is not within the dataset's mountpoint of {}. Path passed: {}".format(ds_mp, path))
raise KeyError(f"Path given is not within the dataset's mountpoint of {ds_mp}. Path passed: {path}")
snap_path = "{}{}".format(snap_path_base, path_neweal.replace(ds_mp, ''))
if os.path.exists(snap_path):
return (True, snap_path)
Expand Down Expand Up @@ -762,17 +769,17 @@ def __init__(self, row, snap_left, snap_right):
self.no_from_snap=True
snap_left = None
elif not isinstance(snap_left, Snapshot):
raise AssertionError("snap_left must be either a Snapshot or str('na-first'). Got: {}".format(type(snap_left)))
raise AssertionError(f"snap_left must be either a Snapshot or str('na-first'). Got: {type(snap_left)}")

if isinstance(snap_right, str) and snap_right == '(present)':
self.to_present=True
snap_right = None

elif not isinstance(snap_right, Snapshot):
raise AssertionError("snap_left must be either a Snapshot. Got: {}".format(type(snap_right)))
raise AssertionError(f"snap_left must be either a Snapshot. Got: {type(snap_right)}")

if not self.no_from_snap and not self.to_present and snap_left.creation >= snap_right.creation:
raise AssertionError("diff from creation ({}) is > or = to diff_to creation ({})".format(snap_left.creation, snap_right.creation))
if not self.no_from_snap and not self.to_present and snap_left.creation > snap_right.creation:
raise AssertionError(f"diff from creation ({snap_left.creation}) is > to diff_to creation ({snap_right.creation})")

self.snap_left = snap_left
self.snap_right = snap_right
Expand All @@ -783,7 +790,7 @@ def __init__(self, row, snap_left, snap_right):
elif len(row) == 5:
(inode_ts, chg_type, file_type, path, path_new) = row
else:
raise Exception("Unexpected len: {}. Row = {}".format(len(row), row))
raise Exception(f"Unexpected len: {len(row)}. Row = {row}")

chg_time = datetime.fromtimestamp(int(inode_ts[:inode_ts.find('.')]))
self.chg_ts = inode_ts
Expand Down Expand Up @@ -834,13 +841,13 @@ def _get_snap_path_right(self):
@staticmethod
def get_file_type(s):
assert (isinstance(s, str) and not s == ''), "argument must be a non-empty string"
assert s in Diff.FILE_TYPES, "ZFS Diff File type is invalid: '{}'".format(s)
assert s in Diff.FILE_TYPES, f"ZFS Diff File type is invalid: '{s}'"
return Diff.FILE_TYPES[s]

@staticmethod
def get_change_type(s):
assert (isinstance(s, str) and not s == ''), "argument must be a non-empty string"
assert s in Diff.CHANGE_TYPES, "ZFS Diff Change type is invalid: '{}'".format(s)
assert s in Diff.CHANGE_TYPES, f"ZFS Diff Change type is invalid: '{s}'"
return Diff.CHANGE_TYPES[s]

def __str__(self):
Expand All @@ -859,57 +866,57 @@ def __str__(self):
# buildTimedelta()
# Builds timedelta from string:
# . tdelta is a timedelta -or- str(nC) where: n is an integer > 0 and C is one of:
# . y=year, m=month, w=week, d=day, H=hour, M=minute, s=second
# . Y=year, M=month, W=week, D=day, h=hour, m=minute, s=second
# Note: month and year are imprecise and assume 30.4 and 365 days
def buildTimedelta(tdelta):
def buildTimedelta(tdelta) -> timedelta:
if isinstance(tdelta, timedelta): return tdelta

if not isinstance(tdelta, str):
raise AssertionError('tdelta must be a string')
raise KeyError('tdelta must be a string')
elif len(tdelta) < 2:
raise AssertionError('len(tdelta) must be >= 2')
raise KeyError('len(tdelta) must be >= 2')
n = tdelta[:-1]
try:
n = int(n)
if n < 1: raise AssertionError('tdelta must be > 0')
if n < 1: raise KeyError('tdelta must be > 0')
except ValueError as ex:
raise AssertionError('Value passed for tdelta does not contain a number: {}'.format(tdelta))
raise KeyError(f'Value passed for tdelta does not contain a number: {tdelta}')

c = tdelta[-1:]
if c == 'H':
if c == 'h':
return timedelta(hours=n)
elif c == 'M':
elif c == 'm':
return timedelta(minutes=n)
elif c == 'S':
elif c == 's':
return timedelta(seconds=n)
elif c == 'd':
elif c == 'D':
return timedelta(days=n)
elif c == 'm':
return timedelta(days=n*(365/12))
elif c == 'w':
elif c == 'W':
return timedelta(weeks=n)
elif c == 'y':
elif c == 'M':
return timedelta(days=n*(365/12))
elif c == 'Y':
return timedelta(days=n*365)
else:
raise AssertionError('Unexpected datetime identifier, expecting one of y,m,w,d,H,M,S.')
raise KeyError('Unexpected datetime identifier, expecting one of Y,M,W,D,h,m,s')


# calcDateRange()
# Calculates a date range based on tdelta string passed
# tdelta is a timedelta -or- str(nC) where: n is an integer > 0 and C is one of:
# . y=year, m=month, w=week, d=day, H=hour, M=minute, s=second
# . Y=year, M=month, W=week, D=day, h=hour, m=minute, s=second
# If dt_from is defined, return tuple: (dt_from, dt_from+tdelta)
# If dt_to is defined, return tuple: (dt_from-tdelta, dt_to)
def calcDateRange(tdelta, dt_from=None, dt_to=None):
if tdelta is None: raise AssertionError('tdelta is required')
def calcDateRange(tdelta, dt_from:datetime=None, dt_to:datetime=None) -> tuple:
if tdelta is None: raise KeyError('tdelta is required')
if dt_from and dt_to:
raise AssertionError('Only one of dt_from or dt_to must be defined')
raise KeyError('Only one of dt_from or dt_to must be defined')
elif (not dt_from and not dt_to):
raise AssertionError('Please specify one of dt_from or dt_to')
raise KeyError('Please specify one of dt_from or dt_to')
elif dt_from and not isinstance(dt_from, datetime):
raise AssertionError('dt_from must be a datetime')
raise KeyError('dt_from must be a datetime')
elif dt_to and not isinstance(dt_to, datetime):
raise AssertionError('dt_to must be a datetime')
raise KeyError('dt_to must be a datetime')

td = buildTimedelta(tdelta)

Expand All @@ -920,7 +927,7 @@ def calcDateRange(tdelta, dt_from=None, dt_to=None):


def splitPath(s):
assert isinstance(s, str), "String not passed. Got: {}".format(type(s))
assert isinstance(s, str), f"String not passed. Got: {type(s)}"
s = s.strip()
assert not s == '', "Empty string passed"
f = os.path.basename(s)
Expand Down Expand Up @@ -976,7 +983,25 @@ def expand_user(path):
return pathlib.Path(path).expanduser()


# Ignore snapshots with exact same timestamp
# . Edge cases that can happen and muck up stuff
# . Handles list(of Snapshot) and list(of tuple(of idx, Snapshot))
def removeDuplicateSnapshotsByDate(snapshots):
_ret = []
for i, snap_rec in enumerate(snapshots):
if isinstance(snap_rec, Snapshot):
snap = snap_rec
elif isinstance(snap_rec, tuple):
(idx, snap) = snap_rec
else:
raise Exception(f"Invalid snapshot list passed. Got {type(snap_rec)} at record {i}")

if i > 0 and snap.creation == snap_last.creation: continue

_ret.append(snap_rec)

snap_last = snap
return _ret


''' END Utilities '''
Expand Down
50 changes: 25 additions & 25 deletions tests/test_zfslib.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ def test_snapable_find_snapshots(self):
snaps = ds.find_snapshots({'name': '*zsys_*e*', 'tdelta': timedelta(hours=36)
,'dt_to': dt_from_creation('1609362247')})
self.assertEqual(len(snaps), 5)
snaps2 = ds.find_snapshots({'name': '*zsys_*e*', 'tdelta': '36H'
snaps2 = ds.find_snapshots({'name': '*zsys_*e*', 'tdelta': '36h'
,'dt_to': dt_from_creation('1609362247')})
self.assertEqual(len(snaps2), len(snaps))
for i, snap in enumerate(snaps):
Expand All @@ -348,7 +348,7 @@ def test_snapable_find_snapshots(self):
,'dt_from': dt_from_creation('1608233673'), 'tdelta': timedelta(hours=48)})
self.assertEqual(len(snaps), 5)
snaps2 = ds.find_snapshots({'name': '*zsys_*w*'
,'dt_from': dt_from_creation('1608233673'), 'tdelta': '48H'})
,'dt_from': dt_from_creation('1608233673'), 'tdelta': '48h'})
self.assertEqual(len(snaps2), len(snaps))
for i, snap in enumerate(snaps):
self.assertIs(snap, snaps2[i])
Expand Down Expand Up @@ -384,15 +384,15 @@ def test_snapable_find_snapshots(self):
with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'dt_from': 'asdf'})
with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'dt_to': 'asdf'})
with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'tdelta': 10})
with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'tdelta': '-10H'})
with self.assertRaises(KeyError): snaps = ds.find_snapshots({'tdelta': '-10h'})
with self.assertRaises(AssertionError): snaps = ds.find_snapshots({'index': 1})
with self.assertRaises(AssertionError):
snaps = ds.find_snapshots({'dt_to': dt_date(2020, 12, 20)
, 'dt_from': dt_date(2020, 12, 21)})
with self.assertRaises(AssertionError):
snaps = ds.find_snapshots({'dt_to': dt_date(2020, 12, 21)
,'dt_from': dt_date(2020, 12, 20)
,'tdelta': "1H"})
,'tdelta': "1h"})


# tested against data_nomounts.tsv
Expand Down Expand Up @@ -422,45 +422,45 @@ def test_no_mounts(self):
# Test General Utilities
def test_buildTimedelta(self):
self.assertEqual(zfs.buildTimedelta(timedelta(seconds=10)), timedelta(seconds=10))
self.assertEqual(zfs.buildTimedelta('1y'), timedelta(days=365))
self.assertEqual(zfs.buildTimedelta('1m'), timedelta(days=(365/12)))
self.assertEqual(zfs.buildTimedelta('1w'), timedelta(weeks=1))
self.assertEqual(zfs.buildTimedelta('10d'), timedelta(days=10))
self.assertEqual(zfs.buildTimedelta('10H'), timedelta(hours=10))
self.assertEqual(zfs.buildTimedelta('10M'), timedelta(minutes=10))
self.assertEqual(zfs.buildTimedelta('10S'), timedelta(seconds=10))
self.assertEqual(zfs.buildTimedelta('1Y'), timedelta(days=365))
self.assertEqual(zfs.buildTimedelta('1M'), timedelta(days=(365/12)))
self.assertEqual(zfs.buildTimedelta('1W'), timedelta(weeks=1))
self.assertEqual(zfs.buildTimedelta('10D'), timedelta(days=10))
self.assertEqual(zfs.buildTimedelta('10h'), timedelta(hours=10))
self.assertEqual(zfs.buildTimedelta('10m'), timedelta(minutes=10))
self.assertEqual(zfs.buildTimedelta('10s'), timedelta(seconds=10))

# Negative tests
with self.assertRaises(TypeError): zfs.buildTimedelta()
with self.assertRaises(TypeError): zfs.buildTimedelta('1H', True)
with self.assertRaises(AssertionError): zfs.buildTimedelta(None)
with self.assertRaises(AssertionError): zfs.buildTimedelta(datetime.now())
with self.assertRaises(AssertionError): zfs.buildTimedelta('1')
with self.assertRaises(AssertionError): zfs.buildTimedelta('aH')
with self.assertRaises(AssertionError): zfs.buildTimedelta('-1H')
with self.assertRaises(AssertionError): zfs.buildTimedelta('1X')
with self.assertRaises(TypeError): zfs.buildTimedelta('1h', True)
with self.assertRaises(KeyError): zfs.buildTimedelta(None)
with self.assertRaises(KeyError): zfs.buildTimedelta(datetime.now())
with self.assertRaises(KeyError): zfs.buildTimedelta('1')
with self.assertRaises(KeyError): zfs.buildTimedelta('ah')
with self.assertRaises(KeyError): zfs.buildTimedelta('-1h')
with self.assertRaises(KeyError): zfs.buildTimedelta('1X')



def test_calcDateRange(self):
_now = datetime.now()
_weekago = datetime.now() - timedelta(weeks=1)
self.assertEqual(zfs.calcDateRange('1d', dt_to=_now), \
self.assertEqual(zfs.calcDateRange('1D', dt_to=_now), \
(_now - timedelta(days=1), _now) )

self.assertEqual(zfs.calcDateRange('2d', dt_from=_weekago), \
self.assertEqual(zfs.calcDateRange('2D', dt_from=_weekago), \
(_weekago, _weekago + timedelta(days=2)) )

self.assertEqual(zfs.calcDateRange('3m', dt_to=_now), \
self.assertEqual(zfs.calcDateRange('3M', dt_to=_now), \
(_now - timedelta(days=(3 * (365/12))), _now) )

# Negative tests
with self.assertRaises(TypeError): zfs.calcDateRange()
with self.assertRaises(TypeError): zfs.calcDateRange(1,2,3,4)
with self.assertRaises(AssertionError): zfs.calcDateRange(None)
with self.assertRaises(AssertionError): zfs.calcDateRange('1H', _now, _now)
with self.assertRaises(AssertionError): zfs.calcDateRange('1H', dt_from=None)
with self.assertRaises(AssertionError): zfs.calcDateRange('1H', dt_to=None)
with self.assertRaises(KeyError): zfs.calcDateRange(None)
with self.assertRaises(KeyError): zfs.calcDateRange('1h', _now, _now)
with self.assertRaises(KeyError): zfs.calcDateRange('1h', dt_from=None)
with self.assertRaises(KeyError): zfs.calcDateRange('1h', dt_to=None)


def test_splitPath(self):
Expand Down
Loading

0 comments on commit 6329927

Please sign in to comment.