diff --git a/ghpages/examples/add_amr_mrgtree.c b/ghpages/examples/add_amr_mrgtree.c index aa28682c..f9afc055 100644 --- a/ghpages/examples/add_amr_mrgtree.c +++ b/ghpages/examples/add_amr_mrgtree.c @@ -110,10 +110,10 @@ Organization: Internet Archive
- The Internet Archive discovers and captures web pages through many different web crawls. - -At any given time several distinct crawls are running, some for months, and some every day or longer. - + The Internet Archive discovers and captures web pages through many different web crawls. + +At any given time several distinct crawls are running, some for months, and some every day or longer. + View the web archive through the Wayback Machine.
@@ -144,4 +144,4 @@ __wm.bt(700,27,25,2,"web","https://wci.llnl.gov/content/assets/docs/simulation/c - \ No newline at end of file + diff --git a/src/fpzip/pcencoder.inl b/src/fpzip/pcencoder.inl index 00c6e2c9..a26c26a8 100644 --- a/src/fpzip/pcencoder.inl +++ b/src/fpzip/pcencoder.inl @@ -1,71 +1,71 @@ -// specialization for small alphabets ----------------------------------------- - -template -class PCencoder { -public: - PCencoder(RCencoder* re, RCmodel*const* rm) : re(re), rm(rm) {} - T encode(T real, T pred, unsigned context = 0); - static const unsigned symbols = 2 * (1 << M::bits) - 1; -private: - static const unsigned bias = (1 << M::bits) - 1; // perfect prediction symbol - M map; // maps T to integer type - RCencoder*const re; // entropy encoder - RCmodel*const* rm; // probability modeler(s) -}; - -// encode narrow range type -template -T PCencoder::encode(T real, T pred, unsigned context) -{ - // map type T to unsigned integer type - typedef typename M::FPZIP_Range_t U; - U r = map.forward(real); - U p = map.forward(pred); - // entropy encode d = r - p - re->encode(bias + r - p, rm[context]); - // return decoded value - return map.inverse(r); -} - -// specialization for large alphabets ----------------------------------------- - -template -class PCencoder { -public: - PCencoder(RCencoder* re, RCmodel*const* rm) : re(re), rm(rm) {} - T encode(T real, T pred, unsigned context = 0); - static const unsigned symbols = 2 * M::bits + 1; -private: - static const unsigned bias = M::bits; // perfect prediction symbol - M map; // maps T to integer type - RCencoder*const re; // entropy encoder - RCmodel*const* rm; // probability modeler(s) -}; - -// encode wide range type -template -T PCencoder::encode(T real, T pred, unsigned context) -{ - // map type T to unsigned integer type - typedef typename M::FPZIP_Range_t U; - U r = map.forward(real); - U p = map.forward(pred); - // compute (-1)^s (2^k + m) = r - p, entropy code (s, k), - // and encode the k-bit number m verbatim - if (p < r) { // underprediction - U d = r - p; - unsigned k = PC::bsr(d); - re->encode(bias + 1 + k, rm[context]); - re->encode(d - (U(1) << k), k); - } - else if (p > r) { // overprediction - U d = p - r; - unsigned k = PC::bsr(d); - re->encode(bias - 1 - k, rm[context]); - re->encode(d - (U(1) << k), k); - } - else // perfect prediction - re->encode(bias, rm[context]); - // return decoded value - return map.inverse(r); -} +// specialization for small alphabets ----------------------------------------- + +template +class PCencoder { +public: + PCencoder(RCencoder* re, RCmodel*const* rm) : re(re), rm(rm) {} + T encode(T real, T pred, unsigned context = 0); + static const unsigned symbols = 2 * (1 << M::bits) - 1; +private: + static const unsigned bias = (1 << M::bits) - 1; // perfect prediction symbol + M map; // maps T to integer type + RCencoder*const re; // entropy encoder + RCmodel*const* rm; // probability modeler(s) +}; + +// encode narrow range type +template +T PCencoder::encode(T real, T pred, unsigned context) +{ + // map type T to unsigned integer type + typedef typename M::FPZIP_Range_t U; + U r = map.forward(real); + U p = map.forward(pred); + // entropy encode d = r - p + re->encode(bias + r - p, rm[context]); + // return decoded value + return map.inverse(r); +} + +// specialization for large alphabets ----------------------------------------- + +template +class PCencoder { +public: + PCencoder(RCencoder* re, RCmodel*const* rm) : re(re), rm(rm) {} + T encode(T real, T pred, unsigned context = 0); + static const unsigned symbols = 2 * M::bits + 1; +private: + static const unsigned bias = M::bits; // perfect prediction symbol + M map; // maps T to integer type + RCencoder*const re; // entropy encoder + RCmodel*const* rm; // probability modeler(s) +}; + +// encode wide range type +template +T PCencoder::encode(T real, T pred, unsigned context) +{ + // map type T to unsigned integer type + typedef typename M::FPZIP_Range_t U; + U r = map.forward(real); + U p = map.forward(pred); + // compute (-1)^s (2^k + m) = r - p, entropy code (s, k), + // and encode the k-bit number m verbatim + if (p < r) { // underprediction + U d = r - p; + unsigned k = PC::bsr(d); + re->encode(bias + 1 + k, rm[context]); + re->encode(d - (U(1) << k), k); + } + else if (p > r) { // overprediction + U d = p - r; + unsigned k = PC::bsr(d); + re->encode(bias - 1 - k, rm[context]); + re->encode(d - (U(1) << k), k); + } + else // perfect prediction + re->encode(bias, rm[context]); + // return decoded value + return map.inverse(r); +} diff --git a/src/hdf5_drv/silo_hdf5.c b/src/hdf5_drv/silo_hdf5.c index 6d576284..e10714e3 100644 --- a/src/hdf5_drv/silo_hdf5.c +++ b/src/hdf5_drv/silo_hdf5.c @@ -2050,6 +2050,9 @@ T_str(char *s) * * Mark C. Miller, Thu Jul 17 23:11:41 PDT 2008 * Added code to detect compression errors. + * + * Mark C. Miller, Wed Aug 26 09:01:49 PDT 2026 + * Handle missing deflate compressor too. *------------------------------------------------------------------------- */ PRIVATE herr_t @@ -2066,6 +2069,8 @@ silo_walk_cb(unsigned n, const H5E_error2_t *err_desc, void *client_data) *silo_error_code_p = E_COMPRESSION; else if (strstr(err_desc->desc, "Lindstrom-") != 0) *silo_error_code_p = E_COMPRESSION; + else if (strstr(err_desc->desc, "deflate") != 0) + *silo_error_code_p = E_COMPRESSION; #if HDF5_VERSION_GE(1,10,0) else if (err_desc->min_num == H5E_CANTLOCKFILE || (strstr(err_desc->desc, "lock") != 0)) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 00512726..795bd698 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -787,7 +787,8 @@ if(SILO_ENABLE_PYTHON_MODULE AND Python_EXECUTABLE) set_tests_properties(test-write-py-hdf5 PROPERTIES RESOURCE_LOCK "sami.silo") set_tests_properties(test-write-py-hdf5 PROPERTIES LABELS "python;hdf5") add_test(NAME test-scale-quadvar-hdf5 COMMAND ${CMAKE_COMMAND} -E env "${silo_test_output_dir}/test_scale_quadvar") - set_tests_properties(test-scale-quadvar-hdf5 PROPERTIES ENVIRONMENT "PYTHONPATH=$${_envvar_path_sep}$ENV{PYTHONPATH};LD_PRELOAD=${SILO_LIBASAN_PATH}:${SILO_LIBUBSAN_PATH};ASAN_OPTIONS=detect_leaks=0") + set_tests_properties(test-scale-quadvar-hdf5 PROPERTIES SKIP_RETURN_CODE ${_silo_test_skip_retval} + ENVIRONMENT "PYTHONPATH=$${_envvar_path_sep}$ENV{PYTHONPATH};LD_PRELOAD=${SILO_LIBASAN_PATH}:${SILO_LIBUBSAN_PATH};ASAN_OPTIONS=detect_leaks=0;SILO_TEST_SKIP_RETURN_CODE=${_silo_test_skip_retval}") set_tests_properties(test-scale-quadvar-hdf5 PROPERTIES LABELS "python;hdf5") endif() diff --git a/tests/scale_quadvar.py b/tests/scale_quadvar.py index 8573debd..42b30803 100755 --- a/tests/scale_quadvar.py +++ b/tests/scale_quadvar.py @@ -1,15 +1,35 @@ #!/usr/bin/env python3 +import os, sys import Silo +# Ensure we can run standalone too +try: + SKIP_RETURN_VALUE = int(os.environ["SILO_TEST_SKIP_RETURN_CODE"]) +except: + SKIP_RETURN_VALUE = 17 + # read box-in-a-box data try: f = Silo.Open("box_in_a_box.silo", Silo.DB_READ) except: f = Silo.Open("bin/box_in_a_box.silo", Silo.DB_READ) -msName = f.GetToc().qmesh_names[0] -msInfo = f.GetVarInfo(msName,1) +# +# The test file used here was produced with a Silo library using +# deflate compression. If the HDF lib in current use is not set +# up for that, reads will fail with compression errors. If that +# happens, just skip this test. If any other read failure occurs +# that is a real problem. +# +try: + msName = f.GetToc().qmesh_names[0] + msInfo = f.GetVarInfo(msName,1) +except Silo.SiloException as e: + if e.args[0] == Silo.E_COMPRESSION: + print("HDF5 lib not configured with deflate filter") + sys.exit(SKIP_RETURN_VALUE) + raise znName = f.GetToc().mat_names[0] znInfo = f.GetVarInfo(znName, 1) diff --git a/tools/python/pydbfile.cpp b/tools/python/pydbfile.cpp index 26bd340b..0574f9bd 100644 --- a/tools/python/pydbfile.cpp +++ b/tools/python/pydbfile.cpp @@ -377,7 +377,8 @@ static PyObject *DBfile_DBGetVarInfo(PyObject *self, PyObject *args) char msg[256]; snprintf(msg, sizeof(msg), "Unable to get component \"%s\" for object \"%s\"", compname.c_str(), str); SiloErrorFunc(msg); - continue; + DBFreeObject(silo_obj); + return NULL; } } @@ -851,8 +852,45 @@ static PyObject *DBfile_DBWriteObject(PyObject *self, PyObject *args) int ncomps = PyDict_Size((PyObject*)dictobj); if (!ncomps) return NULL; - int objtype = DBGetObjtypeTag(PyString_AsString(PyDict_GetItemString((PyObject*)dictobj, "type"))); + + PyObject *typeObj = + PyDict_GetItemString((PyObject*)dictobj, "type"); + + if (!typeObj || !PyString_Check(typeObj)) + { + PyErr_SetString(PyExc_TypeError, + "Object dictionary must contain a string-valued \"type\" entry"); + return NULL; + } + + char const *typeName = PyString_AsString(typeObj); + int objtype = DBGetObjtypeTag(typeName); + DBobject *siloobj = DBMakeObject(objname, objtype, ncomps); + if (!siloobj) + { + SiloErrorFunc("DBMakeObject failed"); + return NULL; + } + + /* + * DBGetObjtypeTag maps arbitrary user-defined object type names + * to DB_USERDEF, while DBMakeObject(DB_USERDEF) assigns the + * canonical type name "unknown". Preserve the original user-defined + * type name here. + */ + if (objtype == DB_USERDEF && strcmp(typeName, "unknown")) + { + FREE(siloobj->type); + siloobj->type = strdup(typeName); + if (!siloobj->type) + { + DBFreeObject(siloobj); + PyErr_NoMemory(); + return NULL; + } + } + PyObject *key, *value; #if PY_VERSION_GE(2,5,0) Py_ssize_t pos = 0; @@ -862,20 +900,20 @@ static PyObject *DBfile_DBWriteObject(PyObject *self, PyObject *args) while (PyDict_Next((PyObject*)dictobj, &pos, &key, &value)) { // skip name and type values - if (!strncmp(PyString_AsString(key), "type", 4)) + if (!strcmp(PyString_AsString(key), "type")) continue; - if (!strncmp(PyString_AsString(key), "name", 4)) + if (!strcmp(PyString_AsString(key), "name")) continue; // handle some special cases Silo treats as either float or double explicitly - if (!strncmp(PyString_AsString(key), "time", 4)) + if (!strcmp(PyString_AsString(key), "time")) { DBAddFltComponent(siloobj, PyString_AsString(key), (float) PyFloat_AS_DOUBLE(value)); continue; } - if (!strncmp(PyString_AsString(key), "missing_value", 13) || - !strncmp(PyString_AsString(key), "extents", 7) || - !strncmp(PyString_AsString(key), "dtime", 5)) + if (!strcmp(PyString_AsString(key), "missing_value") || + !strcmp(PyString_AsString(key), "extents") || + !strcmp(PyString_AsString(key), "dtime")) { DBAddDblComponent(siloobj, PyString_AsString(key), PyFloat_AS_DOUBLE(value)); continue; @@ -965,11 +1003,16 @@ static PyObject *DBfile_DBWriteObject(PyObject *self, PyObject *args) } } - // Ok, we've built up the object, now write it - DBWriteObject(db, siloobj, 1); + + int err = DBWriteObject(db, siloobj, 1); DBFreeObject(siloobj); - PyErr_Clear(); + if (err < 0) + { + SiloErrorFunc("DBWriteObject failed"); + return NULL; + } + Py_INCREF(Py_None); return Py_None; diff --git a/tools/python/pysilo.cpp b/tools/python/pysilo.cpp index 559f2501..8f539e33 100644 --- a/tools/python/pysilo.cpp +++ b/tools/python/pysilo.cpp @@ -101,7 +101,9 @@ std::vector SiloMethods; void SiloErrorFunc(const char *errString) { - PyErr_SetString(SiloError, errString); + PyObject *args = Py_BuildValue("(is)", db_errno, errString); + PyErr_SetObject(SiloError, args); + Py_DECREF(args); } // **************************************************************************** @@ -288,6 +290,45 @@ void SILOMODULE_API initSilo(void) PyDict_SetItemString(d, "SiloException", SiloError); Py_DECREF(SiloError); + // Error codes + ADD_CONSTANT(E_NOERROR) + ADD_CONSTANT(E_BADFTYPE) + ADD_CONSTANT(E_NOTIMP) + ADD_CONSTANT(E_NOFILE) + ADD_CONSTANT(E_INTERNAL) + ADD_CONSTANT(E_NOMEM) + ADD_CONSTANT(E_BADARGS) + ADD_CONSTANT(E_CALLFAIL) + ADD_CONSTANT(E_NOTFOUND) + ADD_CONSTANT(E_TAURSTATE) + ADD_CONSTANT(E_MSERVER) + ADD_CONSTANT(E_PROTO) + ADD_CONSTANT(E_NOTDIR) + ADD_CONSTANT(E_MAXOPEN) + ADD_CONSTANT(E_NOTFILTER) + ADD_CONSTANT(E_MAXFILTERS) + ADD_CONSTANT(E_FEXIST) + ADD_CONSTANT(E_FILEISDIR) + ADD_CONSTANT(E_FILENOREAD) + ADD_CONSTANT(E_SYSTEMERR) + ADD_CONSTANT(E_FILENOWRITE) + ADD_CONSTANT(E_INVALIDNAME) + ADD_CONSTANT(E_NOOVERWRITE) + ADD_CONSTANT(E_CHECKSUM) + ADD_CONSTANT(E_COMPRESSION) + ADD_CONSTANT(E_GRABBED) + ADD_CONSTANT(E_NOTREG) + ADD_CONSTANT(E_CONCURRENT) + ADD_CONSTANT(E_DRVRCANTOPEN) + ADD_CONSTANT(E_BADOPTCLASS) + ADD_CONSTANT(E_NOTENABLEDINBUILD) + ADD_CONSTANT(E_MAXFILEOPTSETS) + ADD_CONSTANT(E_NOHDF5) + ADD_CONSTANT(E_EMPTYOBJECT) + ADD_CONSTANT(E_OBJBUFFULL) + ADD_CONSTANT(E_NOSILOHDF5) + ADD_CONSTANT(E_FILELOCKING) + // Error Modes ADD_CONSTANT(DB_TOP); ADD_CONSTANT(DB_NONE);