8000 Upgrade the nczarr code to match Zarr V2 · DennisHeimbigner/netcdf-c@74b40fd · GitHub
[go: up one dir, main page]

Skip to content

Commit 74b40fd

Browse files
Upgrade the nczarr code to match Zarr V2
Re: zarr-developers/zarr-python#716 The Zarr version 2 spec has been extended to include the ability to choose the dimension separator in chunk name keys. The legal separators has been extended from {'.'} to {'.' '/'}. So now it is possible to use a key like "0/1/2/0" for chunk names. This PR implements this for NCZarr. The V2 spec now says that this separator can be set on a per-variable basis. For now, I have chosen to allow this be set only globally by adding a key named "ZARR.DIMENSION_SEPARATOR=<char>" in the .daprc/.dodsrc/ncrc file. Currently, the only legal separator characters are '.' (the default) and '/'. On writing, this key will only be written if its value is different than the default. This change caused problems because supporting a separator of '/' is difficult to parse when keys/paths use '/' as the path separator. A test case was added for this. Additionally, make nczarr be enabled default by default. This required some additional changes so that if zip and/or AWS S3 sdk are unavailable, then they are disabled for NCZarr. In addition the following unrelated changes were made. 1. Tested that pure-zarr mode could read an nczarr formatted store. 1. The .rc file handling now merges all known .rc files (.ncrc,.daprc, and .dodsrc) in that order and using those in HOME first, then in current directory. For duplicate entries, the later ones override the earlier ones. This change is to remove some of the conflicts inherent in the current .rc file load process. A set of test cases was also added. 1. Re-order tests in configure.ac and CMakeLists.txt so that if libcurl is not found then the other options that depend upon it properly are disabled. 1. I decided that xarray support should be enabled by default for pure zarr. In order to allow disabling, I added a new mode flag "noxarray". 1. Certain test in nczarr_test depend on use of .dodsrc. In order for these to work when testing in parallel, some inter-test dependencies needed to be added. 1. Improved authorization testing to use changes in thredds.ucar.edu
1 parent 706e33b commit 74b40fd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+860
-458
lines changed

.github/workflows/run_tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
name: Run netCDF Tests
66

7-
on: [pull_request]
7+
on: [pull_request,push]
88

99
jobs:
1010

CMakeLists.txt

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,11 +1071,14 @@ IF(NOT FOUND_CURL)
10711071
ENDIF()
10721072

10731073
IF(ENABLE_DAP2 OR ENABLE_DAP4)
1074-
MESSAGE(FATAL_ERROR "DAP support specified, CURL libraries are not found.")
1074+
MESSAGE(WARNING "CURL libraries are not found; DAP support disabled")
1075+
SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE)
1076+
SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE)
10751077
ENDIF()
10761078

10771079
IF(ENABLE_HDF5_ROS3)
1078-
MESSAGE(FATAL_ERROR "ROS3 support specified, CURL libraries are not found.")
1080+
MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.")
1081+
SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
10791082
ENDIF()
10801083

10811084
IF(ENABLE_NCZARR_S3)
@@ -2185,7 +2188,7 @@ is_enabled(STATUS_PNETCDF HAS_PNETCDF)
21852188
is_enabled(STATUS_PARALLEL HAS_PARALLEL)
21862189
is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
21872190
is_enabled(ENABLE_DAP HAS_DAP)
2188-
is_enabled(ENABLE_DAP HAS_DAP2)
2191+
is_enabled(ENABLE_DAP2 HAS_DAP2)
21892192
is_enabled(ENABLE_DAP4 HAS_DAP4)
21902193
is_enabled(ENABLE_BYTERANGE HAS_BYTERANGE)
21912194
is_enabled(ENABLE_DISKLESS HAS_DISKLESS)

configure.ac

Lines changed: 65 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
122122
test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
123123
AC_MSG_RESULT($enable_pnetcdf)
124124

125+
# We need curl for remote operations
126+
AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
127+
if test "x$found_curl" = "xyes" ; then
128+
AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],[])
129+
fi
130+
125131
## Capture the state of the --enable-dap flag => enable dap2+dap4
126132
AC_MSG_CHECKING([whether DAP client(s) are to be built])
127133
AC_ARG_ENABLE([dap],
@@ -130,11 +136,16 @@ AC_ARG_ENABLE([dap],
130136
test "x$enable_dap" = xno || enable_dap=yes
131137
AC_MSG_RESULT($enable_dap)
132138

133-
AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled])
139+
if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then
140+
AC_MSG_WARN([curl required for dap access. DAP support disabled.])
141+
enable_dap=no
142+
fi
143+
144+
AC_MSG_CHECKING([whether netcdf zarr storage format should be disabled])
134145
AC_ARG_ENABLE([nczarr],
135-
[AS_HELP_STRING([--enable-nczarr],
136-
[enable netcdf zarr storage support; requires netCDF-4 and libcurl])])
137-
test "x$enable_nczarr" = xyes || enable_nczarr=no
146+
[AS_HELP_STRING([--disable-nczarr],
147+
[disable netcdf zarr storage support])])
148+
test "x$enable_nczarr" = xno || enable_nczarr=yes
138149
AC_MSG_RESULT($enable_nczarr)
139150

140151
# HDF5 | HDF4 | NCZarr => netcdf-4
@@ -395,12 +406,6 @@ if test "x$enable_set_log_level_func" = xyes -a "x$enable_netcdf_4" = xyes; then
395406
fi
396407
AC_MSG_RESULT($enable_set_log_level_func)
397408

398-
# We need curl for DAP and byterange
399-
AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
400-
if test "x$found_curl" = "xyes" ; then
401-
AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],
402-
[AC_MSG_ERROR([curl required for remote access. Install curl or disable relevant options.])])
403-
fi
404409

405410
# CURLOPT_USERNAME is not defined until curl version 7.19.1
406411
# CURLOPT_PASSWORD is not defined until curl version 7.19.1
@@ -560,63 +565,86 @@ AC_MSG_RESULT([$enable_dap_long_tests])
560565

561566
# Control zarr storage
562567
if test "x$enable_nczarr" = xyes ; then
563-
if test "x$enable_netcdf_4" = xno ; then
564-
AC_MSG_ERROR([netCDF-4 disabled, so you must not enable nczarr])
565-
enable_nczarr=no
566-
fi
568+
if test "x$enable_netcdf_4" = xno ; then
569+
AC_MSG_WARN([netCDF-4 disabled, so you must not enable nczarr])
570+
enable_nczarr=no
571+
fi
567572
fi
573+
568574
if test "x$enable_nczarr" = xyes; then
569575
AC_DEFINE([ENABLE_NCZARR], [1], [if true, build NCZarr Client])
570576
AC_SUBST(ENABLE_NCZARR)
571577
fi
572578
AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
573579

580+
# See if we have libzip
581+
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
582+
if test "x$have_zip" = "xyes" ; then
583+
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], [])
584+
fi
585+
AC_MSG_CHECKING([whether libzip library is available])
586+
AC_MSG_RESULT([${have_zip}])
587+
588+
enable_nczarr_zip=${have_zip} # alias
589+
590+
if test "x$enable_nczarr" = xno ; then
591+
enable_nczarr_zip=no
592+
fi
593+
594+
AC_MSG_CHECKING([whether nczarr zip support is enabled])
595+
AC_MSG_RESULT([${enable_nczarr_zip}])
596+
597+
if test "x$enable_nczarr_zip" = xyes ; then
598+
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
599+
fi
600+
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
601+
574602
# Check for enabling of S3 support
575603
AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
576604
AC_ARG_ENABLE([nczarr-s3],
577605
[AS_HELP_STRING([--enable-nczarr-s3],
578606
[enable netcdf zarr S3 support; make sure to set LDFLAGS])])
579607
test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
580-
AC_MSG_RESULT($enable_nczarr_s3)
581-
582-
# Check for enabling S3 testing
583-
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
584-
AC_ARG_ENABLE([nczarr-s3-tests],
585-
[AS_HELP_STRING([--enable-nczarr-s3-tests],
586-
[enable netcdf zarr S3 testing])])
587-
test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
588-
AC_MSG_RESULT($enable_nczarr_s3_tests)
589-
590-
# Disable S3 tests if S3 support is disabled
591-
if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
592-
AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests])
593-
enable_nczarr_s3_tests=no
608+
if test "x$enable_nczarr" = xno ; then
609+
enable_nczarr_s3=no
594610
fi
611+
AC_MSG_RESULT($enable_nczarr_s3)
595612

596-
# Set default
613+
# Note we check for the library after checking for enable_nczarr_s3
614+
# because for some reason this screws up if we unconditionally test for sdk
615+
# and it is not available. Fix someday
597616
have_aws=no
598-
599617
if test "x$enable_nczarr_s3" = xyes ; then
600618
# See if we have the s3 aws library
601619
# Check for the AWS S3 SDK library
602620
AC_LANG_PUSH([C++])
603621
AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
604622
AC_LANG_POP
623+
fi
624+
605625
AC_MSG_CHECKING([whether AWS S3 SDK library is available])
606626
AC_MSG_RESULT([$have_aws])
607627

608-
if test "x$have_aws" = xyes ; then
628+
if test "x$have_aws" = xno ; then
629+
AC_MSG_WARN([AWS SDK not found; disabling S3 support])
630+
enable_nczarr_s3=no
631+
else
609632
AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
610633
fi
611-
fi
612634
AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
613635

614-
if test "x$have_aws" = xno ; then
615-
if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then
616-
AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests])
636+
# Check for enabling S3 testing
637+
AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
638+
AC_ARG_ENABLE([nczarr-s3-tests],
639+
[AS_HELP_STRING([--enable-nczarr-s3-tests],
640+
[enable netcdf zarr S3 testing])])
641+
test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
642+
AC_MSG_RESULT($enable_nczarr_s3_tests)
643+
644+
# Disable S3 tests if S3 support is disabled
645+
if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
646+
AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests])
617647
enable_nczarr_s3_tests=no
618-
enable_nczarr_s3=no
619-
fi
620648
fi
621649

622650
if test "x$enable_nczarr_s3" = xyes ; then
@@ -634,22 +662,6 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then
634662
fi
635663

636664
# Set default
637-
# See if we have libzip
638-
AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
639-
if test "x$have_zip" = "xyes" ; then
640-
AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [],
641-
[AC_MSG_ERROR([libzip search failed.])])
642-
fi
643-
AC_MSG_CHECKING([whether libzip library is available])
644-
AC_MSG_RESULT([${have_zip}])
645-
646-
enable_nczarr_zip=${have_zip} # alias
647-
648-
if test "x$enable_nczarr_zip" = xyes ; then
649-
AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
650-
fi
651-
AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
652-
653665
# Did the user specify a default cache size for NCZarr?
654666
AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
655667
AC_ARG_WITH([chunk-cache-size-nczarr],

include/ncpathmgr.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,7 @@ EXTERNL int NCaccess(const char* path, int mode);
117117
EXTERNL int NCremove(const char* path);
118118
EXTERNL int NCmkdir(const char* path, int mode);
119119
EXTERNL int NCrmdir(const char* path);
120-
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
121-
EXTERNL char* NCcwd(char* cwdbuf, size_t len);
120+
EXTERNL char* NCgetcwd(char* cwdbuf, size_t len);
122121
#ifdef HAVE_SYS_STAT_H
123122
EXTERNL int NCstat(char* path, struct stat* buf);
124123
#endif

include/ncrc.h

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,11 @@ and accessing rc files (e.g. .daprc).
1616
#include "nclist.h"
1717
#include "ncbytes.h"
1818

19+
/* getenv() keys */
20+
#define NCRCENVIGNORE "NCRCENV_IGNORE"
21+
#define NCRCENVRC "NCRCENV_RC"
22+
23+
1924
typedef struct NCTriple {
2025
char* host; /* combined host:port */
2126
char* key;
@@ -34,31 +39,40 @@ typedef struct NCRCinfo {
3439
typedef struct NCRCglobalstate {
3540
int initialized;
3641
char* tempdir; /* track a usable temp dir */
37-
char* home; /* track $HOME for use in creating $HOME/.oc dir */
42+
char* home; /* track $HOME */
43+
char* cwd; /* track getcwd */
3844
NCRCinfo rcinfo; /* Currently only one rc file per session */
45+
struct GlobalZarr { /* Zarr specific parameters */
46+
char dimension_separator;
47+
} zarr;
3948
} NCRCglobalstate;
4049

4150
/* From drc.c */
42-
extern NCRCglobalstate* ncrc_getglobalstate(void);
43-
extern void ncrc_freeglobalstate(void);
51+
EXTERNL void ncrc_initialize(void);
52+
EXTERNL void ncrc_freeglobalstate(void);
4453
/* read and compile the rc file, if any */
45-
extern int NC_rcload(void);
46-
extern char* NC_rclookup(const char* key, const char* hostport);
47-
extern void NC_rcclear(NCRCinfo* info);
48-
extern int NC_set_rcfile(const char* rcfile);
49-
extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
54+
EXTERNL int NC_rcload(void);
55+
EXTERNL char* NC_rclookup(const char* key, const char* hostport);
56+
EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
57+
58+
/* Following are primarily for debugging */
5059
/* Obtain the count of number of triples */
51-
extern size_t NC_rcfile_length(NCRCinfo*);
60+
EXTERNL size_t NC_rcfile_length(NCRCinfo*);
5261
/* Obtain the ith triple; return NULL if out of range */
53-
extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
62+
EXTERNL NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
63+
64+
/* For internal use */
65+
EXTERNL NCRCglobalstate* ncrc_getglobalstate(void);
66+
EXTERNL void NC_rcclear(NCRCinfo* info);
67+
EXTERNL void NC_rcclear(NCRCinfo* info);
5468

5569
/* From dutil.c (Might later move to e.g. nc.h */
56-
extern int NC__testurl(const char* path, char** basenamep);
57-
extern int NC_isLittleEndian(void);
58-
extern char* NC_entityescape(const char* s);
59-
extern int NC_readfile(const char* filename, NCbytes* content);
60-
extern int NC_writefile(const char* filename, size_t size, void* content);
61-
extern char* NC_mktmp(const char* base);
62-
extern int NC_getmodelist(const char* url, NClist** modelistp);
63-
extern int NC_testmode(const char* path, const char* tag);
70+
EXTERNL int NC__testurl(const char* path, char** basenamep);
71+
EXTERNL int NC_isLittleEndian(void);
72+
EXTERNL char* NC_entityescape(const char* s);
73+
EXTERNL int NC_readfile(const char* filename, NCbytes* content);
74+
EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
75+
EXTERNL char* NC_mktmp(const char* base);
76+
EXTERNL int NC_getmodelist(const char* url, NClist** modelistp);
77+
EXTERNL int NC_testmode(const char* path, const char* tag);
6478
#endif /*NCRC_H*/

libdispatch/ddispatch.c

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ size_t NC_coord_zero[NC_MAX_VAR_DIMS] = {0};
3232
size_t NC_coord_one[NC_MAX_VAR_DIMS] = {1};
3333
ptrdiff_t NC_stride_one[NC_MAX_VAR_DIMS] = {1};
3434

35-
NCRCglobalstate ncrc_globalstate;
36-
3735
/*
3836
static nc_type longtype = (sizeof(long) == sizeof(int)?NC_INT:NC_INT64);
3937
static nc_type ulongtype = (sizeof(unsigned long) == sizeof(unsigned int)?NC_UINT:NC_UINT64);
@@ -81,6 +79,20 @@ NCDISPATCH_initialize(void)
8179
globalstate->home = strdup(home);
8280
}
8381

82+
/* Capture $CWD */
83+
{
84+
char cwdbuf[4096];
85+
86+
cwdbuf[0] = '\0';
87+
(void)NCgetcwd(cwdbuf,sizeof(cwdbuf));
88+
89+
if(strlen(cwdbuf) == 0) {
90+
/* use tempdir */
91+
strcpy(cwdbuf, globalstate->tempdir);
92+
}
93+
globalstate->cwd = strdup(cwdbuf);
94+
}
95+
8496
/* Now load RC File */
8597
status = NC_rcload();
8698
ncloginit();

libdispatch/dinfermodel.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ static const struct MACRODEF {
123123
{"s3","mode","nczarr,s3"},
124124
{"bytes","mode","bytes"},
125125
{"xarray","mode","nczarr,zarr,xarray"},
126+
{"noxarray","mode","nczarr,zarr,noxarray"},
126127
{NULL,NULL,NULL}
127128
};
128129

@@ -133,6 +134,7 @@ static const struct MODEINFER {
133134
} modeinferences[] = {
134135
{"zarr","nczarr"},
135136
{"xarray","zarr"},
137+
{"noxarray","zarr"},
136138
{NULL,NULL}
137139
};
138140

0 commit comments

Comments
 (0)
0