diff --git a/casaconfig/__init__.py b/casaconfig/__init__.py index 840a998..8767e96 100644 --- a/casaconfig/__init__.py +++ b/casaconfig/__init__.py @@ -12,3 +12,4 @@ from .private.set_casacore_path import set_casacore_path from .private.get_config import get_config from .private.get_data_info import get_data_info +from .private.CasaconfigErrors import * diff --git a/casaconfig/__main__.py b/casaconfig/__main__.py index a7f32fb..07f1715 100644 --- a/casaconfig/__main__.py +++ b/casaconfig/__main__.py @@ -44,68 +44,96 @@ config.measurespath = flags.measurespath # watch for measurespath of None, that likely means that casasiteconfig.py is in use and this has not been set. It can't be used then. -if flags.measurespath is None: - print("measurespath has been set to None, likely in casasiteconfig.py.") - print("Either provide a measurespath on the casaconfig command line or edit casasiteconfig.py or other a user config.py to set measurespath to a location.") - sys.exit(1) +try: + if flags.measurespath is None: + print("measurespath has been set to None in the user or site config file.") + print("Either provide a measurespath on the casaconfig command line or edit the user or site config file to set measurespath to a location.") + sys.exit(1) -# do any expanduser and abspath - this is what should be used -measurespath = os.path.abspath(os.path.expanduser(flags.measurespath)) + # do any expanduser and abspath - this is what should be used + measurespath = os.path.abspath(os.path.expanduser(flags.measurespath)) -if flags.currentdata: - if not os.path.exists(measurespath) or not os.path.isdir(measurespath): - print("No data installed at %s. The measurespath does not exist or is not a directory." % measurespath) - else: - from casaconfig import get_data_info - print("current data installed at %s" % measurespath) - dataInfo = get_data_info(measurespath) - - # casarundata - casarunInfo = dataInfo['casarundata'] - if casarunInfo is None or casarunInfo['version'] == "invalid": - print("No casarundata found (missing or unexpected readme.txt contents, not obviously legacy casa data).") - elif casarunInfo['version'] == "unknown": - print("casarundata version is unknown (probably legacy casa data not maintained by casaconfig).") + if flags.currentdata: + if not os.path.exists(measurespath) or not os.path.isdir(measurespath): + print("No data installed at %s. The measurespath does not exist or is not a directory." % measurespath) else: - currentVersion = casarunInfo['version'] - currentDate = casarunInfo['date'] - print('casarundata version %s installed on %s' % (currentVersion, currentDate)) + print("current data installed at %s" % measurespath) + dataInfo = casaconfig.get_data_info(measurespath) + + # casarundata + casarunInfo = dataInfo['casarundata'] + if casarunInfo is None or casarunInfo['version'] == "invalid": + print("No casarundata found (missing readme.txt and not obviously legacy casa data).") + if casarunInfo['version'] == "error": + print("Unexpected casarundata readme.txt content; casarundata should be reinstalled.") + elif casarunInfo['version'] == "unknown": + print("casarundata version is unknown (probably legacy casa data not maintained by casaconfig).") + else: + currentVersion = casarunInfo['version'] + currentDate = casarunInfo['date'] + print('casarundata version %s installed on %s' % (currentVersion, currentDate)) - # measures - measuresInfo = dataInfo['measures'] - if measuresInfo is None or measuresInfo['version'] == "invalid": - print("No measures data found (missing or unexpected readme.txt, not obviously legacy measures data).") - elif measuresInfo['version'] == "unknown": - print("measures version is unknown (probably legacy measures data not maintained by casaconfig).") - else: - currentVersion = measuresInfo['version'] - currentDate = measuresInfo['date'] - print('measures version %s installed on %s' % (currentVersion, currentDate)) + # measures + measuresInfo = dataInfo['measures'] + if measuresInfo is None or measuresInfo['version'] == "invalid": + print("No measures data found (missing readme.txt and not obviously legacy measures data).") + if measuresInfo['version'] == "error": + print("Unexpected measures readme.txt content; measures should be reinstalled.") + elif measuresInfo['version'] == "unknown": + print("measures version is unknown (probably legacy measures data not maintained by casaconfig).") + else: + currentVersion = measuresInfo['version'] + currentDate = measuresInfo['date'] + print('measures version %s installed on %s' % (currentVersion, currentDate)) - # ignore any other arguments -elif flags.summary: - from casaconfig.private.summary import summary - summary(config) - # ignore any other arguments -else: - if flags.referencetesting: - print("reference testing using pull_data and 'release' version into %s" % measurespath) - casaconfig.pull_data(measurespath,'release') - # ignore all other options + # ignore any other arguments + elif flags.summary: + from casaconfig.private.summary import summary + summary(config) + # ignore any other arguments else: - # the update options, update all does everything, no need to invoke anything else - print("Checking for updates into %s" % measurespath) - if flags.updateall: - casaconfig.update_all(measurespath,force=flags.force) + if flags.referencetesting: + print("reference testing using pull_data and 'release' version into %s" % measurespath) + casaconfig.pull_data(measurespath,'release') + # ignore all other options else: - # do any pull_update first - if flags.pulldata: - casaconfig.pull_data(measurespath) - # then data_update, not necessary if pull_data just happened - if flags.dataupdate and not flags.pulldata: - casaconfig.data_update(measurespath, force=flags.force) - # then measures_update - if flags.measuresupdate: - casaconfig.measures_update(measurespath, force=flags.force) + # the update options, update all does everything, no need to invoke anything else + print("Checking for updates into %s" % measurespath) + if flags.updateall: + casaconfig.update_all(measurespath,force=flags.force) + else: + # do any pull_update first + if flags.pulldata: + casaconfig.pull_data(measurespath) + # then data_update, not necessary if pull_data just happened + if flags.dataupdate and not flags.pulldata: + casaconfig.data_update(measurespath, force=flags.force) + # then measures_update + if flags.measuresupdate: + casaconfig.measures_update(measurespath, force=flags.force) + +except casaconfig.UnsetMeasurespath as exc: + # UnsetMeasurespath should not happen because measurespath is checked to not be None above, but just in case + print(str(exc)) + print("This exception should not happen, check the previous messages for additional information and try a different path") + sys.exit(1) + +except casaconfig.BadReadme as exc: + print(str(exc)) + sys.exit(1) + +except casaconfig.RemoteError as exc: + print(str(exc)) + print("This is likely due to no network connection or bad connectivity to a remote site, wait and try again") + sys.exit(1) + +except casaconfig.BadLock as exc: + # the output produced by the update functions is sufficient, just re-echo the exception text itself + print(str(exc)) + sys.exit(1) + +except casaconfig.NotWritable as exc: + print(str(exc)) + sys.exit(1) sys.exit(0) diff --git a/casaconfig/private/CasaconfigErrors.py b/casaconfig/private/CasaconfigErrors.py new file mode 100644 index 0000000..3231ea5 --- /dev/null +++ b/casaconfig/private/CasaconfigErrors.py @@ -0,0 +1,42 @@ +# Copyright 2024 AUI, Inc. Washington DC, USA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +this module will be included in the api +""" + +class AutoUpdatesNotAllowed(Exception): + """Raised when a path does not exist or is not owned by the user""" + pass + +class BadLock(Exception): + """Raised when the lock file is not empty and a lock attempt was made""" + pass + +class BadReadme(Exception): + """Raised when a readme.txt file does not contain the expected contents""" + pass + +class NoReadme(Exception): + """Raised when the readme.txt file is not found at path (path also may not exist)""" + +class RemoteError(Exception): + """Raised when there is an error fetching some remote content""" + pass + +class NotWritable(Exception): + """Raised when the path is not writable by the user""" + +class UnsetMeasurespath(Exception): + """Raised when a path argument is None""" + pass diff --git a/casaconfig/private/config_defaults_static.py b/casaconfig/private/config_defaults_static.py index f820623..2c74c7b 100644 --- a/casaconfig/private/config_defaults_static.py +++ b/casaconfig/private/config_defaults_static.py @@ -51,3 +51,6 @@ # include the user's local site-packages in the python path if True. May conflict with CASA modules user_site = False + +# verbosity level for casaconfig +casaconfig_verbose = 1 diff --git a/casaconfig/private/data_available.py b/casaconfig/private/data_available.py index 6f30bd6..10bec84 100644 --- a/casaconfig/private/data_available.py +++ b/casaconfig/private/data_available.py @@ -29,18 +29,25 @@ def data_available(): changing casaconfig functions that use those tarballs). The full filename is the casarundata version expected in casaconfig functions. - Parameters + Parameters: None - Returns - list - version names returned as list of strings + Returns: + list: version names returned as list of strings + + Raises: + casaconfig.RemoteError: Raised when there is an error fetching some remote content + Exception: Unexpected exception while getting list of available casarundata versions """ import html.parser import urllib.request + import urllib.error import ssl import certifi + + from casaconfig import RemoteError class LinkParser(html.parser.HTMLParser): def reset(self): @@ -67,10 +74,13 @@ def handle_starttag(self, tag, attrs): # return the sorted list, earliest versions are first, newest is last return sorted(parser.rundataList) - + + except urllib.error.URLError as urlerr: + raise RemoteError("Unable to retrieve list of available casarundata versions : " + str(urlerr)) from None + except Exception as exc: - print("ERROR! : unexpected exception while getting list of available casarundata versions") - print("ERROR! : %s" % exc) + msg = "Unexpected exception while getting list of available casarundata versions : " + str(exc) + raise Exception(msg) # nothing to return if it got here, must have been an exception return [] diff --git a/casaconfig/private/data_update.py b/casaconfig/private/data_update.py index 767d505..8529cc5 100644 --- a/casaconfig/private/data_update.py +++ b/casaconfig/private/data_update.py @@ -15,12 +15,14 @@ this module will be included in the api """ -def data_update(path=None, version=None, force=False, logger=None, auto_update_rules=False): +def data_update(path=None, version=None, force=False, logger=None, auto_update_rules=False, verbose=None): """ Check for updates to the installed casarundata and install the update or change to the requested version when appropriate. - If no update is necessary then this function will silently return. + The verbose argument controls the level of information provided when this function when the data + are unchanged for expected reasons. A level of 0 prints and logs nothing. A + value of 1 logs the information and a value of 2 logs and prints the information. The path must contain a previously installed version of casarundata. Use pull_data to install casarundata into a new path (empty or does not exist). @@ -92,35 +94,51 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r **Note:** the most recent casarundata may not include the most recent measures data. A data_update is typically followed by a measures_update. - Parameters - - path (str=None) - Folder path to update. Must contain a valid readme.txt. If not set then config.measurespath is used. - - version (str=None) - Version of casarundata to retrieve (usually in the form of casarundata-x.y.z.tar.gz, see data_available()). Default None retrieves the latest. - - force (bool=False) - If True, always re-download the casarundata. Default False will not download casarundata if updated within the past day unless the version parameter is specified and different from what was last downloaded. - - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. - - auto_update_rules (bool=False) - If True then the user must be the owner of path, version must be None, and force must be False. + Parameters: + path (str=None): Folder path to update. Must contain a valid readme.txt. If not set then config.measurespath is used. + version (str=None): Version of casarundata to retrieve (usually in the form of casarundata-x.y.z.tar.gz, see data_available()). Default None retrieves the latest. + force (bool=False): If True, always re-download the casarundata. Default False will not download casarundata if updated within the past day unless the version parameter is specified and different from what was last downloaded. + logger (casatools.logsink=None): Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. + auto_update_rules (bool=False): If True then the user must be the owner of path, version must be None, and force must be False. + verbose (int): Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal, defaults to casaconfig_verbose in the config dictionary. - Returns + Returns: None + Raises: + AutoUpdatesNotAllowed : raised when path does not exist as a directory or is not owned by the user + BadLock : raised when the lock file was not empty when an attempt was made to obtain the lock + BadReadme : raised when the readme.txt file at path did not contain the expected list of installed files or was incorrectly formatted + NoReadme : raised when the readme.txt file is not found at path (path also may not exist) + NotWritable : raised when the user does not have permission to write to path + RemoteError : raised by data_available when the list of available data versions could not be fetched + UnsetMeasurespath : raised when path is None and measurespath has not been set in config. + Exception: raised when there was an unexpected exception while populating path + """ import os - from .data_available import data_available + from casaconfig import data_available + from casaconfig import pull_data + from casaconfig import get_data_info + from casaconfig import AutoUpdatesNotAllowed, BadReadme, BadLock, NoReadme, RemoteError, UnsetMeasurespath, NotWritable from .print_log_messages import print_log_messages from .get_data_lock import get_data_lock - from .pull_data import pull_data from .do_pull_data import do_pull_data - from .get_data_info import get_data_info if path is None: from .. import config as _config path = _config.measurespath if path is None: - print_log_messages('path is None and has not been set in config.measurespath. Provide a valid path and retry.', logger, True) + raise UnsetMeasurespath('data_update: path is None and has not been set in config.measurespath. Provide a valid path and retry.') return + if verbose is None: + from .. import config as _config + verbose = _config.casaconfig_verbose + # when a specific version is requested then the measures readme.txt that is part of that version # will get a timestamp of now so that default measures updates won't happen for a day unless the # force argument is used for measures_update @@ -131,30 +149,24 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r if auto_update_rules: if version is not None: - print_log_messages('auto_update_rules requires that version be None', logger, True) + print_log_messages('data_update: auto_update_rules requires that version be None', logger, True) return if force: - print_log_messages('force must be False when auto_update_rules is True', logger, True) + print_log_messages('data_update: force must be False when auto_update_rules is True', logger, True) return if (not os.path.isdir(path)) or (os.stat(path).st_uid != os.getuid()): - msgs = [] - msgs.append("Warning: path must exist as a directory and it must be owned by the user, path = %s" % path) - msgs.append("Warning: no data updates are possible on this path by this user.") - print_log_messages(msgs, logger, False) - return + raise AutoUpdatesNotAllowed("data_update: path must exist as a directory and it must be owned by the user, path = %s" % path) if not os.path.exists(readme_path): # path must exist and it must be empty in order to continue if not os.path.exists(path) or (len(os.listdir(path)) > 0): - print_log_messages('No readme.txt file found at path. Nothing updated or checked.', logger, True); - return + raise NoReadme('data_update: no casarundata readme.txt file found at %s. Nothing updated or checked.' % path); # ok to install a fresh copy, use pull_data directly - return pull_data(path,version,force,logger) + return pull_data(path,version,force,logger,verbose) # path must be writable with execute bit set if (not os.access(path, os.W_OK | os.X_OK)) : - print_log_messages('No permission to write to path, cannot update : %s' % path, logger, True) - return + raise NotWritable('data_update: No permission to write to %s, cannot update.' % path) # try and digest the readme file @@ -162,7 +174,8 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r currentVersion = [] currentDate = [] ageRecent = False - + + # already checked that path is OK, type is OK here, no need to trap for exceptions here dataReadmeInfo = get_data_info(path, logger, type='casarundata') if dataReadmeInfo is None or dataReadmeInfo['version'] == 'invalid': @@ -188,20 +201,17 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r if (len(installed_files) == 0): # this shouldn't happen - msgs = [] - msgs.append('The readme.txt file at path did not contain the expected list of installed files') - msgs.append('choose a different path or empty this path and try again using pull_data') - print_log_messages(msgs, logger, True) - # no lock has been set yet, safe to simply return here - return + # no lock has been set yet, safe to raise this exception without worrying about the lock + raise BadReadme('data_update: the readme.txt file at path did not contain the expected list of installed files') if version is None and force is False and ageRecent: # if version is None, the readme is less than 1 day old and force is False then return without checking for any newer versions - # normal use is silent, this line is useful during debugging - # print_log_messages('data_update latest version checked recently in %s, using version %s' % (path, currentVersion), logger) + if verbose > 0: + print_log_messages('data_update: version installed or checked less than 1 day ago, nothing updated or checked', logger, verbose=verbose) # no lock has been set yet, safe to simply return here return + # this may raise a RemoteError, no need to catch that here but it may need to be caught upstream available_data = data_available() requestedVersion = version latestVersion = False @@ -238,19 +248,20 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r force = False # if measuresReadmeInfo is None then that's a problem and force remains True, this also catches 'invalid' and 'unknown' measures versions, which should not happen here if not force: - # normal use is silent, this line is useful during debugging - # print_log_messages('data_update requested "release" version of casarundata and measures are already installed.', logger) + if verbose > 0: + print_log_messages('data_update: requested "release" version of casarundata and measures are already installed.', logger, verbose=verbose) # no lock has been set yet, safe to simply return here return else: # normal usage, ok to return now - # normal use is silent, commented out lines are useful during debugging if latestVersion: - # print_log_messages('The latest version is already installed in %s, using version %s' % (path, currentVersion), logger) + if verbose > 0: + print_log_messages('The latest version is already installed in %s' % path, logger, verbose=verbose) # touch the dates of the readme to prevent a future check on available data for the next 24 hours os.utime(readme_path) - #else: - # print_log_messages('Requested casarundata is installed in %s, using version %s' % (path, currentVersion), logger) + else: + if verbose > 0: + print_log_messages('Requested casarundata version is already installed in %s, %s' % (path, currentVersion), logger, verbose=verbose) # no lock has been set yet, safe to simply return here return @@ -258,19 +269,12 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r # an update appears necessary lock_fd = None + clean_lock = True # set to False if the contents are actively being update and the lock file should not be cleaned on exception try: print_log_messages('data_update using version %s, acquiring the lock ... ' % requestedVersion, logger) lock_fd = get_data_lock(path, 'data_update') - # if lock_fd is None it means the lock file was not empty - because we know that path exists at this point - if lock_fd is None: - msgs = [] - msgs.append('The lock file at %s is not empty.' % path) - msgs.append('A previous attempt to update path may have failed or exited prematurely.') - msgs.append('Remove the lock file and set force to True with the desired version (default to most recent).') - msgs.append('It may be best to completely repopulate path using pull_data and measures_update.') - print_log_messages(msgs, logger, True) - return + # the BadLock exception that may happen here is caught below do_update = True # it's possible that another process had path locked and updated the readme with new information, re-read it @@ -290,58 +294,71 @@ def data_update(path=None, version=None, force=False, logger=None, auto_update_r do_update = False # if measuresReadmeInfo is None there was a problem which requires a full update so do_update remains True if not do_update: + # always verbose here because the lock file is in use print_log_messages('data update requested "release" version of casarundata and measures are already installed.', logger) else: # nothing to do here, already at the expected version and an update is not being forced if latestVersion: + # always verbose here because the lock file is in use print_log_messages('The latest version is already installed, using version %s' % currentVersion, logger) # touch the dates of the readme to prevent a future check on available data for the next 24 hours os.utime(readme_path) else: + # always verbose here because the lock file is in use print_log_messages('requested version is already installed.', logger) do_update = False if do_update: # update is still on, check the manifest if len(installed_files) == 0: - # this shouldn't happen, do not do an update - do_update = False - msgs = [] - msgs.append('The readme.txt file read at path did not contain the expected list of installed files') - msgs.append('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence') - msgs.append('Check for other updates in process or choose a different path or clear out this path and try again using pull_data or update_all') - print_log_messages(msgs, logger, True) + # this shouldn't happen, do not do an update, raise the BadReadme exception, caught below and the lock cleaned up then + raise BadReadme('data_update: the readme.txt file at path did not contain the expected list of installed files') else: - # this shouldn't happen, do not do an update - do_update = False - msgs = [] - msgs.append('Unexpected problem reading readme.txt file during data_update, can not safely update to the requested version') - msgs.append('This should not happen unless multiple sessions are trying to update at the same time and one experienced problems or was done out of sequence') - msgs.append('Check for other updates in process or choose a different path or clear out this path and try again using pull_data or update_all') - print_log_messages(msgs, logger, True) + # this shouldn't happen + raise BadReadme('data_update: unexpected problem reading readme.txt file during data update, can not safely update to the requested version') if do_update: + # do not clean the lock file contents at this point unless do_pull_data returns normally + clean_lock = False do_pull_data(path, requestedVersion, installed_files, currentVersion, currentDate, logger) + clean_lock = True if namedVersion: # a specific version has been requested, set the times on the measures readme.txt to now to avoid # a default update of the measures data without using the force argument measuresReadmePath = os.path.join(path,'geodetic/readme.txt') os.utime(measuresReadmePath) - # truncate the lock file - lock_fd.truncate(0) + except BadLock as exc: + # the path is known to exist so this means that the lock file was not empty and it's not locked + msgs = str(exc) + msgs.append('data_update: the lock file at %s is not empty.' % path) + msgs.append('A previous attempt to update path may have failed or exited prematurely.') + msgs.append('Remove the lock file and set force to True with the desired version (default to most recent).') + msgs.append('It may be best to completely re-populate path using pull_data and measures_update.') + print_log_messages(msgs, logger, True) + # reraise this + raise + + except BadReadme as exc: + # something is wrong in the readme after an update was triggered, this shouldn't happen, print more context reraise this + msgs = str(exc) + msgs.append('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence') + msgs.append('Check for other updates in progress or choose a different path or clear out this path and try again using pull_data or update_all') + print_log_messages(msgs, logger, True) + raise except Exception as exc: msgs = [] msgs.append('ERROR! : Unexpected exception while populating casarundata version %s to %s' % (requestedVersion, path)) msgs.append('ERROR! : %s' % exc) print_log_messages(msgs, logger, True) - # leave the contents of the lock file as is to aid in debugging - # import traceback - # traceback.print_exc() - - # if the lock file is not closed, do that now to release the lock - if lock_fd is not None and not lock_fd.closed: - lock_fd.close() + raise + + finally: + # make sure the lock file is closed and also clean the lock file if safe to do so, this is always executed + if lock_fd is not None and not lock_fd.closed: + if clean_lock: + lock_fd.truncate(0) + lock_fd.close() return diff --git a/casaconfig/private/do_auto_updates.py b/casaconfig/private/do_auto_updates.py index 25f7f7d..b44d235 100644 --- a/casaconfig/private/do_auto_updates.py +++ b/casaconfig/private/do_auto_updates.py @@ -15,7 +15,7 @@ this module will be included in the api """ -def do_auto_updates(configDict, logger=None): +def do_auto_updates(configDict, logger=None, verbose=None): """ Use measurespath, data_auto_update, and measures_auto_update from configDict to do any auto updates as necessary. @@ -33,41 +33,45 @@ def do_auto_updates(configDict, logger=None): See the documentation for data_update and measures_update for additional details about the auto update rules. - Paramters - - configDict (dict) - A config dictionary previously set. - - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. + The verbose argument controls the level of information provided when this function when the data + are unchanged for expected reasons. A level of 0 prints and logs nothing. A + value of 1 logs the information and a value of 2 logs and prints the information. - Returns + See data_update and measures_update for additional details about exceptions + + Paramters: + configDict (dict): A config dictionary previously set. + logger (casatools.logsink=None): Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. + verbose (int): Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal, defaults to casaconfig_verbose in the config dictionary. + + Returns: None + Raises: + UnsetMeasurespath: raised when measurespath is None in config + """ from .print_log_messages import print_log_messages from .data_update import data_update from .measures_update import measures_update + from casaconfig import UnsetMeasurespath + if configDict.measurespath is None: # continue, because things still might work if there are measures in datapath - msgs = [] - msgs.append('measurespath is None in config') - msgs.append('set measurespath in your config file at ~/.casa/config.py') - msgs.append('or ask the site manager to set that in a casasiteconfig.py') - msgs.append('visit https://casadocs.readthedocs.io/en/stable/notebooks/external-data.html for more information') - - if (configDict.measures_auto_update or configDict.data_auto_update): - msgs.append('Auto updates of measures path are not possible because measurespath is not set, skipping auto updates') - - print_log_messages(msgs, logger, True) + raise UnsetMeasurespath('do_auto_updates: measurespath is None in configDict. Provide a valid path and retry.') - return + if verbose is None: + verbose = configDict.casaconfig_verbose if (configDict.measures_auto_update or configDict.data_auto_update): if (configDict.data_auto_update and (not configDict.measures_auto_update)): print_log_messages('measures_auto_update must be True when data_auto_update is True, skipping auto updates', logger, True) else: if configDict.data_auto_update: - data_update(configDict.measurespath, logger=logger, auto_update_rules=True) + data_update(configDict.measurespath, logger=logger, auto_update_rules=True, verbose=verbose) if configDict.data_auto_update or configDict.measures_auto_update: - measures_update(configDict.measurespath, logger=logger, auto_update_rules=True) + measures_update(configDict.measurespath, logger=logger, auto_update_rules=True, verbose=verbose) return diff --git a/casaconfig/private/do_pull_data.py b/casaconfig/private/do_pull_data.py index e61ebd9..59342cd 100644 --- a/casaconfig/private/do_pull_data.py +++ b/casaconfig/private/do_pull_data.py @@ -23,15 +23,15 @@ def do_pull_data(path, version, installed_files, currentVersion, currentDate, lo calling function has already examined any existing readme file and used that to set the installed_files list as appropriate. - Parameters - - path (str) - Folder path to place casadata contents. - - version (str) - casadata version to retrieve. - - installed_files (str list) - list of installed files from the version already installed. Set to an empty list if there is no previously installed version. - - currentVersion (str) - from the readme file if it already exists, or an empty string if there is no previously installed version. - - currentDate (str) - from the readme file if it already exists, or an empty string if there is no previously installed version. - - logger (casatools.logsink) - Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Set to None to skip writing messages to a logger. + Parameters: + path (str): Folder path to place casadata contents. + version (str): casadata version to retrieve. + installed_files (str list): list of installed files from the version already installed. Set to an empty list if there is no previously installed version. + currentVersion (str): from the readme file if it already exists, or an empty string if there is no previously installed version. + currentDate (str): from the readme file if it already exists, or an empty string if there is no previously installed version. + logger (casatools.logsink): Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Set to None to skip writing messages to a logger. - Returns + Returns: None """ diff --git a/casaconfig/private/get_config.py b/casaconfig/private/get_config.py index dc0d056..daaa59b 100644 --- a/casaconfig/private/get_config.py +++ b/casaconfig/private/get_config.py @@ -22,11 +22,11 @@ def get_config( default=False ): The default values (returned when default is True) are the configuration values after all config files have been evaluated but before the path values have been expanded using os.path.expanduser and os.path.abspath. Modules that use the command line to change config values may also not update the default values. User actions in a CASA session will also typically not change the default values. - Parameters - default (bool=False) - If True, return the default values. + Parameters: + default (bool=False): If True, return the default values. - Returns - list[str] - list of configuration strings + Returns: + list[str]: list of configuration strings """ from .. import config as _config diff --git a/casaconfig/private/get_data_info.py b/casaconfig/private/get_data_info.py index 6cf9eaf..3acda44 100644 --- a/casaconfig/private/get_data_info.py +++ b/casaconfig/private/get_data_info.py @@ -70,16 +70,20 @@ def get_data_info(path=None, logger=None, type=None): If path has not been set (has a value of None) then the returned value will be None. This likely means that a casasiteconfig.py exists but has not yet been edited to set measurespath. - Parameters - - path (str) - Folder path to find the casarundata and measures data information. If not set then config.measurespath is used. - - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. - - type (str) - the specific type of data info to return (None, 'casarundata', 'measures', 'release'; None returns a dictionary of all types) + Parameters: + path (str): Folder path to find the casarundata and measures data information. If not set then config.measurespath is used. + logger (casatools.logsink=None): Instance of the casalogger to use for writing messages. Default None writes messages to the terminal. + type (str): the specific type of data info to return (None, 'casarundata', 'measures', 'release'; None returns a dictionary of all types) - Returns - dict - a dictionary by type, 'casarundata', 'measures', 'release' where each type is a dictionary containing 'version' and 'date'. A return value of None indicates path is unset. A value of None for that type means no information could be found about that type. If a specific type is requested then only the dictionary for that type is returned (or None if that type can not be found). + Returns: + dict: a dictionary by type, 'casarundata', 'measures', 'release' where each type is a dictionary containing 'version' and 'date'. A return value of None indicates path is unset. A value of None for that type means no information could be found about that type. If a specific type is requested then only the dictionary for that type is returned (or None if that type can not be found). + + Raises: + UnsetMeasurespath: path is None and has not been set in config + ValueError: raised when type has an invalid value + """ - # when None is returned, path wasn't set result = None import os @@ -87,6 +91,9 @@ def get_data_info(path=None, logger=None, type=None): import importlib.resources from .print_log_messages import print_log_messages from .read_readme import read_readme + + from casaconfig import UnsetMeasurespath + currentTime = time.time() secondsPerDay = 24. * 60. * 60. @@ -96,8 +103,7 @@ def get_data_info(path=None, logger=None, type=None): path = _config.measurespath if path is None: - print_log_messages('path is None and has not been set in config.measurespath. Provide a valid path and retry.', logger, True) - return None + raise UnsetMeasurespath('get_data_info: path is None and has not been set in config.measurespath. Provide a valid path and retry.') path = os.path.abspath(os.path.expanduser(path)) diff --git a/casaconfig/private/get_data_lock.py b/casaconfig/private/get_data_lock.py index 7a9597f..a5a8398 100644 --- a/casaconfig/private/get_data_lock.py +++ b/casaconfig/private/get_data_lock.py @@ -16,8 +16,8 @@ def get_data_lock(path, fn_name): """ Get and initialize and set the lock on 'data_update.log' in path. - If path does not already exist and the lock file is not empty then a lock - is not set and the returned value is None. + If path does not already exist or the lock file is not empty then a lock + is not set then a BadLock exception is raised. When a lock is set the lock file will contain the user, hostname, pid, date, and time. @@ -29,11 +29,16 @@ def get_data_lock(path, fn_name): This function is intended for internal casaconfig use. - Parameters - - path (str) - The location where 'data_update.log' is to be found. - - fn_name (str) - A string giving the name of the calling function to be recorded in the lock file. - Returns - - the open file descriptor holding the lock. Close this file descriptor to release the lock. Returns None if path does not exist or the lock file is not empty. + Parameters: + path (str): The location where 'data_update.log' is to be found. + fn_name (str): A string giving the name of the calling function to be recorded in the lock file. + + Returns: + fd: the open file descriptor holding the lock. Close this file descriptor to release the lock. + + Raises: + BadLock: raised when the path to the lock file does not exist or the lock file is not empty as found + Exception: an unexpected exception was seen while writing the lock information to the file """ @@ -42,7 +47,10 @@ def get_data_lock(path, fn_name): import getpass from datetime import datetime - if not os.path.exists(path): return None + from casaconfig import BadLock + + if not os.path.exists(path): + raise BadLock("path to contain lock file does not exist : %s" % path) lock_path = os.path.join(path, 'data_update.lock') lock_exists = os.path.exists(lock_path) @@ -58,7 +66,7 @@ def get_data_lock(path, fn_name): if (len(lockLines) > 0) and (len(lockLines[0]) > 0): # not empty lock_fd.close() - return None + raise BadLock("lock file is not empty : %s" % lock_path) # write the lock information, the seek and truncate are probably not necessary try: @@ -71,7 +79,8 @@ def get_data_lock(path, fn_name): print("ERROR! Called by function : %s" % fn_name) print("ERROR! : %s" % exc) lock_fd.close() - return None + # reraise the exception - this shouldn't happen + raise exc return lock_fd diff --git a/casaconfig/private/measures_available.py b/casaconfig/private/measures_available.py index 2cdcd04..0ee8fce 100644 --- a/casaconfig/private/measures_available.py +++ b/casaconfig/private/measures_available.py @@ -25,20 +25,35 @@ def measures_available(): of the values in that list if set (otherwise the most recent version in this list is used). - Parameters + Parameters: None - Returns - list - version names returned as list of strings + Returns: + list: version names returned as list of strings + + Raises: + RemoteError: raised when when a socket.gaierror is seen, likely due to no network connection + Exception: raised when any unexpected exception happens """ from ftplib import FTP - - ftp = FTP('ftp.astron.nl') - rc = ftp.login() - rc = ftp.cwd('outgoing/Measures') - files = ftp.nlst() - ftp.quit() - #files = [ff.replace('WSRT_Measures','').replace('.ztar','').replace('_','') for ff in files] - files = [ff for ff in files if (len(ff) > 0) and (not ff.endswith('.dat'))] + import socket + + from casaconfig import RemoteError + + files = [] + try: + ftp = FTP('ftp.astron.nl') + rc = ftp.login() + rc = ftp.cwd('outgoing/Measures') + files = ftp.nlst() + ftp.quit() + #files = [ff.replace('WSRT_Measures','').replace('.ztar','').replace('_','') for ff in files] + files = [ff for ff in files if (len(ff) > 0) and (not ff.endswith('.dat'))] + except socket.gaierror as gaierr: + raise RemoteError("Unable to retrieve list of available measures versions : " + str(gaierr)) from None + except Exception as exc: + msg = "Unexpected exception while getting list of available measures versions : " + str(exc) + raise Exception(msg) + return files diff --git a/casaconfig/private/measures_update.py b/casaconfig/private/measures_update.py index 63efb52..03a063e 100644 --- a/casaconfig/private/measures_update.py +++ b/casaconfig/private/measures_update.py @@ -15,7 +15,7 @@ this module will be included in the api """ -def measures_update(path=None, version=None, force=False, logger=None, auto_update_rules=False, use_astron_obs_table=False): +def measures_update(path=None, version=None, force=False, logger=None, auto_update_rules=False, use_astron_obs_table=False, verbose=None): """ Update or install the IERS data used for measures calculations from the ASTRON server at path. @@ -23,6 +23,10 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda If no update is necessary then this function will silently return. + The verbose argument controls the level of information provided when this function when the data + are unchanged for expected reasons. A level of 0 prints and logs nothing. A + value of 1 logs the information and a value of 2 logs and prints the information. + CASA maintains a separate Observatories table which is available in the casarundata collection through pull_data and data_update. The Observatories table found at ASTRON is not installed by measures_update and any Observatories file at path will not be changed @@ -98,16 +102,26 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda is not empty or the readme.txt file can not be read. This use of force should be used with caution. - - Parameters - - path (str=None) - Folder path to place updated measures data. Must contain a valid geodetic/readme.txt. If not set then config.measurespath is used. - - version (str=None) - Version of measures data to retrieve (usually in the form of yyyymmdd-160001.ztar, see measures_available()). Default None retrieves the latest. - - force (bool=False) - If True, always re-download the measures data. Default False will not download measures data if updated within the past day unless the version parameter is specified and different from what was last downloaded. - - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Default None writes messages to the terminal - - auto_update_rules (bool=False) - If True then the user must be the owner of path, version must be None, and force must be False. + Parameters: + path (str=None): Folder path to place updated measures data. Must contain a valid geodetic/readme.txt. If not set then config.measurespath is used. + version (str=None): Version of measures data to retrieve (usually in the form of yyyymmdd-160001.ztar, see measures_available()). Default None retrieves the latest. + force (bool=False): If True, always re-download the measures data. Default False will not download measures data if updated within the past day unless the version parameter is specified and different from what was last downloaded. + logger (casatools.logsink=None): Instance of the casalogger to use for writing messages. Default None writes messages to the terminal + auto_update_rules (bool=False): If True then the user must be the owner of path, version must be None, and force must be False. + verbose (int=None): Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal, defaults to casaconfig_verbose in config dictionary. - Returns + Returns: None + + Raises: + AutoUpdatesNotAllowed: raised when path does not exists as a directory or is not owned by the user when auto_update_rules is True + BadLock: raised when the lock file was not empty when found + BadReadme: raised when something unexpected is found in the readme or the readme changed after an update is in progress + NoReadme : raised when the readme.txt file is not found at path (path also may not exist) + NotWritable: raised when the user does not have permission to write to path + RemoteError: raised by measures_available when the remote list of measures could not be fetched + UnsetMeasurespath: raised when path is None and has not been set in config + Exception: raised when something unexpected happened while updating measures """ import os @@ -123,6 +137,9 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda import certifi import fcntl + from casaconfig import measures_available + from casaconfig import AutoUpdatesNotAllowed, UnsetMeasurespath, RemoteError, NotWritable, BadReadme, BadLock + from .print_log_messages import print_log_messages from .get_data_lock import get_data_lock from .get_data_info import get_data_info @@ -133,8 +150,11 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda path = _config.measurespath if path is None: - print_log_messages('path is None and has not been set in config.measurespath. Provide a valid path and retry.', logger, True) - return + raise UnsetMeasurespath('measures_update: path is None and has not been set in config.measurespath. Provide a valid path and retry.') + + if verbose is None: + from .. import config as _config + verbose = _config.casaconfig_verbose path = os.path.expanduser(path) @@ -146,11 +166,7 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda print_log_messages('force must be False when auto_update_rules is True', logger, True) return if (not os.path.isdir(path)) or (os.stat(path).st_uid != os.getuid()): - msgs = [] - msgs.append("Warning: path must exist as a directory and it must be owned by the user, path = %s" % path) - msgs.append("Warning: no measures auto update is possible on this path by this user.") - print_log_messages(msgs, logger, False) - return + raise AutoUpdatesNotAllowed("measures_update: path must exist as a directory and it must be owned by the user, path = %s" % path) if not os.path.exists(path): # make dirs all the way down, if possible @@ -171,18 +187,20 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda if not force: # don't check for new version if the age is less than 1 day if version is None and ageRecent: - # normal use is silent, this line is useful during debugging - # print_log_messages('measures_update latest version checked recently in %s, using version %s' % (path, current), logger) + if verbose > 0: + print_log_messages('measures_update: version installed or checked less than 1 day ago, nothing updated or checked', logger, verbose=verbose) return # don't overwrite something that looks bad unless forced to do so if current == 'invalid': - print_log_messages('The measures readme.txt file could not be read as expected, an update can not proceed unless force is True', logger, True) - return + raise NoReadme('measures_update: no measures readme.txt file found at %s. Nothing updated or checked.' % path) + + if current == 'error': + raise BadReadme('measures_update: the measures readme.txt file at %s could not be read as expected, an update can not proceed unless force is True' % path) # don't overwrite something that looks like valid measures data unless forced to do so if current == 'unknown': - print_log_messages('The measures data at %s is not maintained by casaconfig and so it can not be updated unless force is True' % path, logger, True) + print_log_messages('measures_update: the measures data at %s is not maintained by casaconfig and so it can not be updated unless force is True' % path, logger) return checkVersion = version @@ -190,22 +208,21 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda # get the current most recent version try: checkVersion = measures_available()[-1] + except RemoteError as exc: + # no network, no point in continuing, just reraise + raise exc except: # unsure what happened, leave it at none, which will trigger an update attempt, which might work pass # don't re-download the same data if (checkVersion is not None) and (checkVersion == current): - # normal use is silent, this line is useful during debugging - # print_log_messages('measures_update requested version already installed in %s' % path, logger) + if verbose > 0: + print_log_messages('measures_update: requested version already installed in %s' % path, logger, verbose=verbose) # update the age of the readme to now - try: - readme_path = os.path.join(path,'geodetic/readme.txt') - # readme_path should already exist if it's here - os.utime(readme_path) - except: - # unsure what happened, everything otherwise is fine if we got here, ignore this error - pass + readme_path = os.path.join(path,'geodetic/readme.txt') + # readme_path should already exist if it's here + os.utime(readme_path) return @@ -221,30 +238,21 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda # path must be writable with execute bit set if (not os.access(path, os.W_OK | os.X_OK)) : - print_log_messages('No permission to write to the measures path, cannot update : %s' % path, logger, True) - return + raise NotWritable('measures_update: No permission to write to path, cannot update : %s' % path) # an update needs to happen # lock the measures_update.lock file lock_fd = None + clean_lock = True # set to false if the contents are actively being u pdate and the lock file should not be cleaned one exception try: print_log_messages('measures_update ... acquiring the lock ... ', logger) + # the BadLock exception that may happen here is caught below lock_fd = get_data_lock(path, 'measures_update') - # if lock_fd is None it means the lock file was not empty - because we know that path exists at this point - if lock_fd is None: - # using a list of messages results in a better printing if the logger is redirected to the terminal - msgs = [] - msgs.append('The lock file at %s is not empty.' % path) - msgs.append('A previous attempt to update path may have failed or exited prematurely.') - msgs.append('Remove the lock file and set force to True with the desired version (default to most recent).') - msgs.append('It may be best to completely repopulate path using pull_data and measures_update.') - print_log_messages(msgs, logger, True) - return do_update = force - + if not do_update: # recheck the readme file, another update may have already happened before the lock was obtained current = None @@ -257,22 +265,19 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda ageRecent = readmeInfo['age'] < 1.0 if (version is not None) and (version == current): - # no update will be done, version is as requested - not silent here because the lock is in use + # no update will be done, version is as requested - always verbose here because the lock is in use print_log_messages('The requested measures version is already installed in %s, using version %s' % (path, current), logger) elif (version is None) and ageRecent: - # no update will be done, it's already been checked or updated recently - not silent here because the lock is in use + # no update will be done, it's already been checked or updated recently - always verbose here because the lock is in use print_log_messages('The latest measures version was checked recently in %s, using version %s' % (path, current), logger) else: # final check for problems before updating if not force and readmeInfo is not None and (version=='invalid' or version=='unknown'): # at this point, this indicates something is unexpectedly wrong, do not continue - # using a list of messages results in a better printing if the logger is redirected to the terminal - msgs = [] - msgs.append('Something unexpected has changed in the measures path location, and measures_update can not continue') - msgs.append('A previous measures_update may have exited unexpectedly') - msgs.append('It may be necessary to reinstall the casarundata as well as the measures data if %s is the correct path' % path) - print_log_messages(msgs, logger, True) - # update is already turned off, the lock file will be cleaned up on exit + # this exception is caught below + # do not clean up the lock file + clean_lock = False + raise BadReadme('measures_update: something unexpected has changed in the path location, can not continue') else: # an update is needed do_update = True @@ -283,6 +288,7 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda print_log_messages(' ... connecting to ftp.astron.nl ...', logger) + clean_lock = False ftp = FTP('ftp.astron.nl') rc = ftp.login() rc = ftp.cwd('outgoing/Measures') @@ -293,7 +299,7 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda # but that isn't checked - this could install a version that's already installed target = files[-1] if version is None else version if target not in files: - print_log_messages('measures_update cant find specified version %s' % target, logger, True) + print_log_messages("measures_update can't find specified version %s" % target, logger, True) else: # there are files to extract, make sure there's no past measures.ztar from a failed previous install @@ -337,23 +343,45 @@ def measures_update(path=None, version=None, force=False, logger=None, auto_upda with open(readme_path,'w') as fid: fid.write("# measures data populated by casaconfig\nversion : %s\ndate : %s" % (target, datetime.today().strftime('%Y-%m-%d'))) - print_log_messages(' ... measures data update at %s' % path, logger) + print_log_messages(' ... measures data updated at %s' % path, logger) + + clean_lock = True # closing out the do_update # closing out the try block - # truncate the lock file - lock_fd.truncate(0) + + except BadLock as exc: + # the path is known to exist so this means that the lock file was not empty and it's not locked + msgs = [str(exc)] + msgs.append('The lock file at %s is not empty.' % path) + msgs.append('A previous attempt to update path may have failed or exited prematurely.') + msgs.append('Remove the lock file and set force to True with the desired version (default to most recent).') + msgs.append('It may be best to completely repopulate path using pull_data and measures_update.') + print_log_messages(msgs, logger, True) + # reraise this + raise + + except BadReadme as exc: + # something is wrong in the readme after an update was triggered, this shouldn't happen, print more context and reraise this + msgs = [str(exc)] + msgs.append('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence') + msgs.append('Check for other updates in progress or choose a different path or clear out this path and reinstall the casarundata as well as the measures data') + print_log_messages(msgs, logger, True) + raise except Exception as exc: msgs = [] msgs.append("ERROR! : Unexpected exception while updating measures at %s" % path) msgs.append("ERROR! : %s" % exc) print_log_messages(msgs, logger, True) - # leave the contents of the lock file as is to aid in debugging - - # if the lock file is not closed, do that now to release the lock - if lock_fd is not None and not lock_fd.closed: - lock_fd.close() + raise + + finally: + # make sure the lock file is closed and also clean the lock file if safe to do so, this is always executed + if lock_fd is not None and not lock_fd.closed: + if clean_lock: + lock_fd.truncate(0) + lock_fd.close() return diff --git a/casaconfig/private/print_log_messages.py b/casaconfig/private/print_log_messages.py index 7f9989d..a478981 100644 --- a/casaconfig/private/print_log_messages.py +++ b/casaconfig/private/print_log_messages.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -def print_log_messages(msg, logger, is_err=False): +def print_log_messages(msg, logger, is_err=False, verbose=2): """ Print msg and optionally write it to an instance of the casalogger. @@ -22,12 +22,15 @@ def print_log_messages(msg, logger, is_err=False): When is_err is True the message is printed sys.stderr and logged as SEVERE + When verbose is < 2 then the message is not printed unless there is no logger + This function is intended for internal casaconfig use. Parameters - - msg (str) - The message to print and optionally log. - - logger (casatools.logsink) - Instance of the casalogger to use. Not used if None. - - is_err (bool=False) - When False, output goes to sys.stdout and logged as INFO level. When True, output goes to sys.stderr and logged as SEVERE + msg (str): The message to print and optionally log. + logger (casatools.logsink): Instance of the casalogger to use. Not used if None. + is_err (bool=False): When False, output goes to sys.stdout and logged as INFO level. When True, output goes to sys.stderr and logged as SEVERE + verbose (int=2): When < 2 then msg is only printed if there is no logger, otherwise it's just logged Returns None @@ -45,8 +48,10 @@ def print_log_messages(msg, logger, is_err=False): if type(msg) is not list: msg = [msg] - for m_msg in msg: - print(m_msg,file=fileout) + # always print if there is no logger, if there is a logger and verbose is < 2 then do not print + if (logger is None) or (not verbose < 2): + for m_msg in msg: + print(m_msg,file=fileout) for m_msg in msg: if logger is not None: logger.post(m_msg, loglevel) diff --git a/casaconfig/private/pull_data.py b/casaconfig/private/pull_data.py index 30e15e8..032c9f7 100644 --- a/casaconfig/private/pull_data.py +++ b/casaconfig/private/pull_data.py @@ -15,10 +15,14 @@ this module will be included in the api """ -def pull_data(path=None, version=None, force=False, logger=None): +def pull_data(path=None, version=None, force=False, logger=None, verbose=None): """ Pull the casarundata contents from the CASA host and install it in path. + The verbose argument controls the level of information provided when this function when the data + are unchanged for expected reasons. A level of 0 prints and logs nothing. A + value of 1 logs the information and a value of 2 logs and prints the information. + The path must either contain a previously installed version of casarundata or it must not exist or be empty. @@ -79,32 +83,41 @@ def pull_data(path=None, version=None, force=False, logger=None): what versions are available. There is no check on when the data were last updated before calling data_available (as there is in the two update functions). - Parameters - - path (str) - Folder path to place casarundata contents. It must be empty or not exist or contain a valid, previously installed version. If not set then config.measurespath is used. - - version (str=None) - casadata version to retrieve. Default None gets the most recent version. - - force (bool=False) - If True, re-download and install the data even when the requested version matches what is already installed. Default False will not download data if the installed version matches the requested version. - - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Default None does not write any messages to a logger. + Parameters: + path (str): Folder path to place casarundata contents. It must be empty or not exist or contain a valid, previously installed version. If not set then config.measurespath is used. + version (str=None): casadata version to retrieve. Default None gets the most recent version. + force (bool=False): If True, re-download and install the data even when the requested version matches what is already installed. Default False will not download data if the installed version matches the requested version. + logger (casatools.logsink=None): Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Default None does not write any messages to a logger. + verbose (int): Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal, defaults to casaconfig_verbose in the config dictionary. - Returns + Returns: None + Raises: + BadLock: raised when the lock file is not empty when a lock is requested + BadReadme: raised when the readme.txt file found at path does not contain the expected list of installed files or there was an unexpected change while the data lock is on + NotWritable: raised when the user does not have write permission to path + RemoteError : raised by data_available when the list of available data versions could not be fetched + UnsetMeasurespath: raised when path is None and and measurespath has not been set in config. + """ import os - from .data_available import data_available + from casaconfig import data_available + from casaconfig import get_data_info + from casaconfig import UnsetMeasurespath, BadLock, BadReadme, NotWritable + from .print_log_messages import print_log_messages from .get_data_lock import get_data_lock from .do_pull_data import do_pull_data - from .get_data_info import get_data_info if path is None: from .. import config as _config path = _config.measurespath if path is None: - print_log_messages('path is None and has not been set in config.measurespath. Provide a valid path and retry.', logger, True) - return + raise UnsetMeasurespath('path is None and has not been set in config.measurespath. Provide a valid path and retry.') # when a specific version is requested then the measures readme.txt that is part of that version # will get a timestamp of now so that default measures updates won't happen for a day unless the @@ -149,16 +162,13 @@ def pull_data(path=None, version=None, force=False, logger=None): if (installed_files is None or len(installed_files) == 0): # this shouldn't happen - msgs = [] - msgs.append('destination path is not empty and the readme.txt file found there did not contain the expected list of installed files') - msgs.append('choose a different path or empty this path and try again') - print_log_messages(msgs, logger, True) - # no lock as been set yet, safe to simply return here - return + # no lock has been set yet, safe to raise this exception without worrying about the lock + raise BadReadme('pull_data: the readme.txt file at path did not contain the expected list of installed files') # the readme file looks as expected, pull if the version is different or force is true if version is None: # use most recent available + # this may raise a RemoteError, no need to catch that here but it may need to be caught upstream available_data = data_available() version = available_data[-1] @@ -167,11 +177,13 @@ def pull_data(path=None, version=None, force=False, logger=None): if not do_pull: # it's already at the expected version and force is False, nothing to do # safe to return here, no lock has been set + print_log_messages('pull_data: version is already at the expected version and force is False. Nothing was changed', logger, verbose=verbose) return # a pull will happen, unless the version string is not available if available_data is None: + # this may raise a RemoteError, no need to catch that here but it may need to be caught upstream available_data = data_available() if version is None: @@ -196,21 +208,18 @@ def pull_data(path=None, version=None, force=False, logger=None): # make dirs all the way down path if possible os.makedirs(path) + # path must be writable with execute bit set + if (not os.access(path, os.W_OK | os.X_OK)) : + raise NotWritable('pull_data: No permission to write to path, cannot update : %s' % path) + # lock the data_update.lock file lock_fd = None + clean_lock = True # set to False if the contents are actively being update and the lock file should not be cleaned on exception try: print_log_messages('pull_data using version %s, acquiring the lock ... ' % version, logger) lock_fd = get_data_lock(path, 'pull_data') - # if lock_fd is None it means the lock file was not empty - because we know that path exists at this point - if lock_fd is None: - msgs = [] - msgs.append('The lock file at %s is not empty.' % path) - msgs.append('A previous attempt to update path may have failed or exited prematurely.') - msgs.append('Remove the lock file and set force to True with the desired version (default to the most recent).') - msgs.append('It may be best to clean out that location and do a fresh pull_data.') - print_log_messages(msgs, logger, True) - return + # the BadLock exception that may happen here is caught below do_pull = True if not force: @@ -242,48 +251,59 @@ def pull_data(path=None, version=None, force=False, logger=None): # a version of 'invalid', 'error', or 'unknown' is a surprise here, likely caused by something else doing something # incompatible with this attempt if version in ['invalid','error','unknown']: - do_pull = False - msgs = [] - msgs.append('Unexpected version or problem found in readme.txt file during pull_data, can not safely pull the requested version') - msgs.append('This should not happen unless multiple sessions are trying to pull_data at the same time and one experienced problems or was done out of sequence') - print_log_messages(msgs, logger, True) - + # raise BadReadme (caught below) and do not clean up the lock file + clean_lock = False + raise BadReadme('data_update : something unexpected has changed in the path location, can not continue') if do_pull: # make sure the copy of installed_files is the correct one installed_files = readmeInfo['manifest'] if len(installed_files) == 0: - # this shoudn't happen, do not do a pull - do_pull = False - msgs = [] - msgs.append('destination path is not empty and the readme.txt file found there did not contain the expected list of installed files') - msgs.append('This should not happen unless multiple sessions are trying to pull_data at the same time and one experienced problems or was done out of sequence') - msgs.append('Check for other updates in process or choose a different path or clear out this path and try again') - print_log_messages(msgs, logger, True) + # this shoudn't happen, raise BadReadme (caught below) and do not clean up the lock file + clean_lock = False + raise BadReadme('pull_data : the readme.txt file at path did not contain the expected list of installed files after the lock was obtained, this should never happen.') if do_pull: + # do not clean the lock file contents at this point unless do_pull_data returns normally + clean_lock = False do_pull_data(path, version, installed_files, currentVersion, currentDate, logger) + clean_lock = True if namedVersion: # a specific version has been requested, set the times on the measures readme.txt to now to avoid # a default update of the measures data without using the force argument measuresReadmePath = os.path.join(path,'geodetic/readme.txt') os.utime(measuresReadmePath) - - # truncate the lock filed - lock_fd.truncate(0) - + + except BadLock as exc: + # the path is known to exist so this means that the lock file was not empty and it's not locked + msgs = str(exc) + msgs.append('The lock file at %s is not empty.' % path) + msgs.append('A previous attempt to update path may have failed or exited prematurely.') + msgs.append('It may be best to completely repopulated path using pull_data and measures_update.') + print_log_messages(msgs, logger, True) + # reraise this + raise + except BadReadme as exc: + # something is wrong in the readme after an update was triggered and locked, this shouldn't happen, print more context and reraise this + msgs = str(exc) + msgs.append('This should not happen unless multiple sessions are trying to update data at the same time and one experienced problems or was done out of sequence') + msgs.append('Check for other updates in progress or choose a different path or clear out this path and try again') + print_log_messages(msgs, logger, True) + raise + except Exception as exc: msgs = [] msgs.append('ERROR! : Unexpected exception while populating casarundata version %s to %s' % (version, path)) msgs.append('ERROR! : %s' % exc) print_log_messages(msgs, logger, True) - # leave the contents of the lock file as is to aid in debugging - # import traceback - # traceback.print_exc() - - # if the lock file is not closed, do that now to release the lock - if lock_fd is not None and not lock_fd.closed: - lock_fd.close() + raise + + finally: + # make sure the lock file is closed and also clean the lock file if safe to do so, this is always executed + if lock_fd is not None and not lock_fd.closed: + if clean_lock: + lock_fd.truncate(0) + lock_fd.close() return diff --git a/casaconfig/private/read_readme.py b/casaconfig/private/read_readme.py index f036081..d247d44 100644 --- a/casaconfig/private/read_readme.py +++ b/casaconfig/private/read_readme.py @@ -29,12 +29,12 @@ def read_readme(path): The version string and date are stripped of leading and trailing whitespace. - Parameters - - path (str) - the path to the file to be read + Parameters: + path (str): the path to the file to be read - Returns - Dictionary of 'version' (the version string), 'date' (the date string), - 'extra' (a list of any extra lines found). The return value is None on error. + Returns: + dict: Dictionary of 'version' (the version string), 'date' (the date string), + 'extra' (a list of any extra lines found). The return value is None on error. """ import os @@ -55,6 +55,6 @@ def read_readme(path): extra.append(extraLine.strip()) result = {'version':version, 'date':date, 'extra':extra} except: - pass + result = None return result diff --git a/casaconfig/private/set_casacore_path.py b/casaconfig/private/set_casacore_path.py index bb50fb1..2047965 100644 --- a/casaconfig/private/set_casacore_path.py +++ b/casaconfig/private/set_casacore_path.py @@ -25,10 +25,10 @@ def set_casacore_path(path=None): the casacore data directory can be found by casacore users. It sets the value of that parameter to the casacore data location, replacing any previously set value. - Parameters - - path (string=None) - path to the desired data directory. Default None uses the included data directory from this package + Parameters: + path (string=None): path to the desired data directory. Default None uses the included data directory from this package - Returns + Returns: None """ diff --git a/casaconfig/private/summary.py b/casaconfig/private/summary.py index 25bdb02..d743103 100644 --- a/casaconfig/private/summary.py +++ b/casaconfig/private/summary.py @@ -22,11 +22,25 @@ def summary(configDict = None): value of the measurespath config value is shown, the installed data versions are shown (casarundata and measures) and any release version information is shown. + Parameters: + configDict (dict): a config dictionary. If None this is imported from casaconfig. + + Returns: + None + """ import casaconfig - dataInfo = casaconfig.get_data_info() + if configDict is None: + from casaconfig import config as configDict + + try: + dataInfo = casaconfig.get_data_info() + except casaconfig.UnsetMeasurespath: + print("Measurespath is unset or does not exist in config dictionary. Use a valid path and try again") + return + print("") print("casaconfig summary") print("") @@ -54,6 +68,8 @@ def summary(configDict = None): msg = "casarundata version : %s" % rundataVers if rundataVers == "unknown": msg += "; legacy data not maintained by casaconfig" + elif rundataVers == "error": + msg += "; unexpected readme.txt file, casarundata should be reinstalled" print(msg) if (dataInfo['measures'] is None): print("measures version : no measures found") @@ -62,6 +78,8 @@ def summary(configDict = None): msg = "measures version : %s" % measVers if measVers == "unknown": msg += "; legacy data not maintained by casaconfig" + elif measVers == "error": + msg += "; unexpected readme.txt file, measures should be reinstalled" print(msg) if (dataInfo['release'] is None): diff --git a/casaconfig/private/update_all.py b/casaconfig/private/update_all.py index f839a58..59295a3 100644 --- a/casaconfig/private/update_all.py +++ b/casaconfig/private/update_all.py @@ -15,7 +15,7 @@ this module will be included in the api """ -def update_all(path=None, logger=None, force=False): +def update_all(path=None, logger=None, force=False, verbose=None): """ Update the data contants at path to the most recently released versions of casarundata and measures data. @@ -35,18 +35,24 @@ def update_all(path=None, logger=None, force=False): The force argument is passed to data_update and measures_update + The verbose argument controls the level of information provided when + this function when the data are unchanged for expected reasons. A level + of 0 prints and logs nothing. A value of 1 logs the information and a value + of 2 logs and prints the information. + This uses pull_data, data_update and measures_update. See the - documentation for those functions for additional details. + documentation for those functions for additional details (e.g. verbose argument). Some of the data updated by this function is only read when casatools starts. Use of update_all after CASA has started should typically be followed by a restart so that any changes are seen by the tools and tasks that use this data. - Parameters - - path (str=None) - Folder path to place casarundata contents. It must not exist, or be empty, or contain a valid, previously installed version. If it exists, it must be owned by the user. Default None uses the value of measurespath set by importing config.py. - - logger (casatools.logsink=None) - Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Default None does not write any messages to a logger. + Parameters: + path (str=None): Folder path to place casarundata contents. It must not exist, or be empty, or contain a valid, previously installed version. If it exists, it must be owned by the user. Default None uses the value of measurespath set by importing config.py. + logger (casatools.logsink=None): Instance of the casalogger to use for writing messages. Messages are always written to the terminal. Default None does not write any messages to a logger. + verbose (int): Level of output, 0 is none, 1 is to logger, 2 is to logger and terminal, defaults to casaconfig_verbose in the config dictionary. - Returns + Returns: None """ @@ -86,7 +92,7 @@ def update_all(path=None, logger=None, force=False): # if path is empty, first use pull_data if len(os.listdir(path))==0: - pull_data(path, logger) + pull_data(path, logger, verbose=verbose) # double check that it's not empty if len(os.listdir(path))==0: print_log_messages("pull_data failed, see the error messages for more details. update_all can not continue") @@ -102,16 +108,16 @@ def update_all(path=None, logger=None, force=False): print_log_messages('readme.txt not found at path, update_all can not continue, path = %s' % path, logger) return - if dataInfo['casarundata'] is 'invalid': + if dataInfo['casarundata'] == 'invalid': print_log_messages('readme.txt is invalid at path, update_all can not continue, path = %s' % path, logger) return - if dataInfo['casarundata'] is 'unknown': + if dataInfo['casarundata'] == 'unknown': print_log_messages('contents at path appear to be casarundata but no readme.txt was found, casaconfig did not populate this data and update_all can not continue, path = %s', path, logger) return # the updates should work now - data_update(path, logger, force=force) - measures_update(path, logger, force=force) + data_update(path, logger, force=force, verbose=verbose) + measures_update(path, logger, force=force, verbose=verbose) return diff --git a/tests/test_casaconfig.py b/tests/test_casaconfig.py index 5f7d31a..d304b98 100644 --- a/tests/test_casaconfig.py +++ b/tests/test_casaconfig.py @@ -107,17 +107,13 @@ def test_file_exists(self): '''Test Default config.py exists in casaconfig module''' self.assertTrue(os.path.isfile('{}/casaconfig/config.py'.format(sitepackages))) + @unittest.skipIf(not os.path.exists(os.path.join(sitepackages,'casatools')), "casatools not found") def test_import_casatools_bad_measurespath(self): '''Test that import casatools will return ImportError with measurespath set to an empty directory that cannot be written to''' # Due to casatools / casaconfig caching, run the import of casatools and read stdout/stderr for ImportError # after first creating an empty measurespath directory with the write permissions turned off # and then creating a test config file using the path to that directory as measurespath - # skip this test if casatools is not available (testing casaconfig apart from the rest of casa) - if not os.path.exists(os.path.join(sitepackages,'casatools')): - print("casatools is not available in sitepackages. skipping test_import_casatools_bad_measurespath") - return - if (not os.path.exists(self.emptyPath)): os.mkdir(self.emptyPath) @@ -137,8 +133,10 @@ def test_import_casatools_bad_measurespath(self): (output, _) = proc.communicate() p_status = proc.wait() - ref = True if "ImportError" in str(output) else False - self.assertTrue(ref, "ImportError Not Found") + print("test_import_casatools_bad_measurespath") + print(str(output)) + ref = True if "NotWritable" in str(output) else False + self.assertTrue(ref, "NotWritable Not Found") def test_casaconfig_measures_available(self): '''Test That Today or Yesterday measures data is returned''' @@ -171,14 +169,10 @@ def test_casaconfig_measures_update(self): self.assertTrue(newVers == vers) + @unittest.skipIf(not os.path.exists(os.path.join(sitepackages,'casatools')), "casatools not found") def test_read_measurespath_from_user_config(self): '''Test casaconfig downloads specific measures data to location and casatools reads that data location''' - # skip this test if casatools is not available (testing casaconfig apart from the rest of casa) - if not os.path.exists(os.path.join(sitepackages,'casatools')): - print("casatools is not available in sitepackages. skipping test_read_measurespath_from_user_config") - return - # this requires that there be the full casarundata already installed self.populate_testrundata() @@ -212,20 +206,18 @@ def test_read_measurespath_from_user_config(self): p_status = proc.wait() ref = False if "AssertionError" in str(output) else True + print("test_read_measurespath_from_user_config") + print(str(output)) self.assertTrue(ref, "AssertionError seen in output : expected utils().measurespath() was not seen") # final check that the expected version is now installed installedVers = casaconfig.get_data_info(path=self.testRundataPath,type='measures')['version'] self.assertTrue(installedVers == vers, "expected version was not installed : %s != %s" % (installedVers, vers)) + @unittest.skipIf(not os.path.exists(os.path.join(sitepackages,'casatools')), "casatools not found") def test_auto_update_measures(self): '''Test Automatic Measures Updates to measurespath''' - # skip this test if casatools is not available (testing casaconfig apart from the rest of casa) - if not os.path.exists(os.path.join(sitepackages,'casatools')): - print("casatools is not available in sitepackages. skipping test_auto_update_measures") - return - # this requires that there be the full casarundata already installed self.populate_testrundata() @@ -259,16 +251,14 @@ def test_auto_update_measures(self): # output should contain the latest version string ref = self.get_meas_avail()[-1] in str(output) + print("test_auto_update_measures") + print(str(output)) self.assertTrue(ref, "Update Failed") + @unittest.skipIf(not os.path.exists(os.path.join(sitepackages,'casatools')), "casatools not found") def test_auto_install_data(self): '''Test auto install of all data to measurespath on casatools startup''' - # skip this test if casatools is not available (testing casaconfig apart from the rest of casa) - if not os.path.exists(os.path.join(sitepackages,'casatools')): - print("casatools is not available in sitepackages. skipping test_auto_update_measures") - return - # make sure that testrundata does not exist if os.path.exists(self.testRundataPath): shutil.rmtree(self.testRundataPath) @@ -286,8 +276,10 @@ def test_auto_install_data(self): (output, _) = proc.communicate() p_status = proc.wait() - ref = True if "ImportError" in str(output) else False - self.assertTrue(ref, "ImportError Not Found") + ref = True if "AutoUpdatesNotAllowed" in str(output) else False + print("test_auto_install_data, 1") + print(str(output)) + self.assertTrue(ref, "AutoUpdatesNotAllowed not found") # create testRundataPath and try again os.mkdir(self.testRundataPath) @@ -295,6 +287,8 @@ def test_auto_install_data(self): (output, _) = proc.communicate() p_status = proc.wait() + print("test_auto_install_data, 2") + print(str(output)) ref = True if "ImportError" not in str(output) else False self.assertTrue(ref, "ImportError Found") @@ -303,6 +297,8 @@ def test_auto_install_data(self): expectedDataVersion = casaconfig.data_available()[-1] expectedMeasVersion = self.get_meas_avail()[-1] ref = (dataInfo['casarundata']['version'] == expectedDataVersion) and (dataInfo['measures']['version'] == expectedMeasVersion) + print("test_auto_install_data, 3") + print(str(output)) self.assertTrue(ref, "Expected versions not installed") def test_daily_update(self): @@ -355,9 +351,6 @@ def test_daily_update(self): self.assertTrue((checkRundataVers == rundataVers) and (checkMeasVers != measVers), "versions are not as expected after a measures update") - # remember this measures version - measVers = checkMeasVers - # backdate the rundata rundataReadmePath = os.path.join(self.testRundataPath,'readme.txt') os.utime(rundataReadmePath,(olderTime,olderTime)) @@ -372,7 +365,11 @@ def test_daily_update(self): dataInfo = casaconfig.get_data_info(self.testRundataPath) checkRundataVers = dataInfo['casarundata']['version'] checkMeasVers = dataInfo['measures']['version'] - self.assertTrue((checkRundataVers != rundataVers) and (checkMeasVers == measVers), "versions are not as expected after a measures update") + + # IF a new measures tarball was made available while this test was running then the updated measure here may be one more than the previous + # update, so long as this is the most recent measures this test is OK + expectedMeasVers = casaconfig.measures_available()[-1] + self.assertTrue((checkRundataVers != rundataVers) and (checkMeasVers == expectedMeasVers), "versions are not as expected after a measures update") def do_config_check(self, expectedDict, noconfig, nositeconfig): '''Launch a separate python to load the config files, using --noconfig --nositeconfig as requested, the expectedDict contains expected values'''