Skip to content

Commit

Permalink
Merge pull request #11601 from rouault/GDAL_NETCDF_REPORT_EXTRA_DIM_V…
Browse files Browse the repository at this point in the history
…ALUES_fix

netCDF: fix 3.10.1RC1 regression regarding reporting of values of extra dimensions
  • Loading branch information
rouault authored Jan 8, 2025
2 parents bf234c3 + fc26671 commit 1d3a440
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 29 deletions.
Binary file not shown.
90 changes: 90 additions & 0 deletions autotest/gdrivers/netcdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -6634,3 +6634,93 @@ def test_netcdf_var_with_geoloc_array_but_no_coordinates_attr():
got_md["X_DATASET"]
== 'NETCDF:"data/netcdf/var_with_geoloc_array_but_no_coordinates_attr.nc":lon'
)


###############################################################################
# Test reporting of values of an extra dimension which is unlimited


def test_netcdf_var_extra_dim_unlimited():

ds = gdal.Open("data/netcdf/extra_dim_unlimited.nc")
assert ds.GetMetadataItem("NETCDF_DIM_time_VALUES") == "{17927,17955}"

with gdal.config_option("GDAL_NETCDF_REPORT_EXTRA_DIM_VALUES", "YES"):
ds = gdal.Open("data/netcdf/extra_dim_unlimited.nc")
assert ds.GetMetadataItem("NETCDF_DIM_time_VALUES") == "{17927,17955}"

with gdal.config_option("GDAL_NETCDF_REPORT_EXTRA_DIM_VALUES", "NO"):
ds = gdal.Open("data/netcdf/extra_dim_unlimited.nc")
assert ds.GetMetadataItem("NETCDF_DIM_time_VALUES") is None


###############################################################################
# Test reporting of values of an extra dimension which is unlimited


@pytest.mark.require_curl()
@pytest.mark.skipif(sys.platform != "linux", reason="Incorrect platform")
def test_netcdf_var_extra_dim_unlimited_network():

import webserver

webserver_process = None
webserver_port = 0

(webserver_process, webserver_port) = webserver.launch(
handler=webserver.DispatcherHttpHandler
)
if webserver_port == 0:
pytest.skip()

filename = "data/netcdf/extra_dim_unlimited.nc"

gdal.VSICurlClearCache()

try:
filesize = gdal.VSIStatL(filename).size
handler = webserver.SequentialHandler()
handler.add("HEAD", "/test.nc", 200, {"Content-Length": "%d" % filesize})

def method(request):
# sys.stderr.write('%s\n' % str(request.headers))

if request.headers["Range"].startswith("bytes="):
rng = request.headers["Range"][len("bytes=") :]
assert len(rng.split("-")) == 2
start = int(rng.split("-")[0])
end = int(rng.split("-")[1])

request.protocol_version = "HTTP/1.1"
request.send_response(206)
request.send_header("Content-type", "application/octet-stream")
request.send_header(
"Content-Range", "bytes %d-%d/%d" % (start, end, filesize)
)
request.send_header("Content-Length", end - start + 1)
request.send_header("Connection", "close")
request.end_headers()
with open(filename, "rb") as f:
f.seek(start, 0)
request.wfile.write(f.read(end - start + 1))

handler.add("GET", "/test.nc", custom_method=method)

handler.add("HEAD", "/test.nc.aux.xml", 404)
handler.add("GET", "/", 404)
handler.add("HEAD", "/test.aux", 404)
handler.add("HEAD", "/test.AUX", 404)
handler.add("HEAD", "/test.nc.aux", 404)
handler.add("HEAD", "/test.nc.AUX", 404)

with webserver.install_http_handler(handler):
with gdal.quiet_errors():
ds = gdal.Open("/vsicurl/http://127.0.0.1:%d/test.nc" % webserver_port)
if ds is None:
pytest.skip("cannot open /vsicurl/ netCDF file")
assert ds.GetMetadataItem("NETCDF_DIM_time_VALUES") is None

finally:
webserver.server_stop(webserver_process, webserver_port)

gdal.VSICurlClearCache()
51 changes: 22 additions & 29 deletions frmts/netcdf/netcdfdataset.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8861,40 +8861,33 @@ GDALDataset *netCDFDataset::Open(GDALOpenInfo *poOpenInfo)
// dimension in its NETCDF_DIM_xxxx band metadata item
// Addresses use case of
// https://lists.osgeo.org/pipermail/gdal-dev/2023-May/057209.html
const bool bIsLocal =
VSIIsLocal(osFilenameForNCOpen.c_str());
bool bListDimValues =
lev_count == 1 ||
bIsLocal || lev_count == 1 ||
!NCDFIsUnlimitedDim(poDS->eFormat ==
NCDF_FORMAT_NC4,
cdfid, poDS->m_anDimIds[j]);
if (!bListDimValues &&
!VSIIsLocal(osFilenameForNCOpen.c_str()))
const char *pszGDAL_NETCDF_REPORT_EXTRA_DIM_VALUES =
CPLGetConfigOption(
"GDAL_NETCDF_REPORT_EXTRA_DIM_VALUES", nullptr);
if (pszGDAL_NETCDF_REPORT_EXTRA_DIM_VALUES)
{
const char *pszGDAL_NETCDF_REPORT_EXTRA_DIM_VALUES =
CPLGetConfigOption(
"GDAL_NETCDF_REPORT_EXTRA_DIM_VALUES",
nullptr);
if (!pszGDAL_NETCDF_REPORT_EXTRA_DIM_VALUES)
{
if (!bREPORT_EXTRA_DIM_VALUESWarningEmitted)
{
bREPORT_EXTRA_DIM_VALUESWarningEmitted =
true;
CPLDebug(
"GDAL_netCDF",
"Listing extra dimension values is "
"skipped because this dataset is "
"hosted on a network file system, and "
"such an operation could be slow. If "
"you still want to proceed, set the "
"GDAL_NETCDF_REPORT_EXTRA_DIM_VALUES "
"configuration option to YES");
}
}
else
{
bListDimValues = CPLTestBool(
pszGDAL_NETCDF_REPORT_EXTRA_DIM_VALUES);
}
bListDimValues = CPLTestBool(
pszGDAL_NETCDF_REPORT_EXTRA_DIM_VALUES);
}
else if (!bListDimValues && !bIsLocal &&
!bREPORT_EXTRA_DIM_VALUESWarningEmitted)
{
bREPORT_EXTRA_DIM_VALUESWarningEmitted = true;
CPLDebug(
"GDAL_netCDF",
"Listing extra dimension values is skipped "
"because this dataset is hosted on a network "
"file system, and such an operation could be "
"slow. If you still want to proceed, set the "
"GDAL_NETCDF_REPORT_EXTRA_DIM_VALUES "
"configuration option to YES");
}
if (bListDimValues)
{
Expand Down

0 comments on commit 1d3a440

Please sign in to comment.