From: Egbert Eich <eich@suse.com>
Date: Wed May 22 10:01:50 2024 +0200
Subject: Add unused bits checks and tools patches
Patch-mainline: Upstream
Git-repo: https://github.com/HDFGroup/hdf5
Git-commit: 7d9d4988e9edd7b692eb8513eb3c3a7f8609e2b7
References: bsc#1224158

NOTE: Requires fixup!
* Sizes are tweaked by one byte for unknown reasons

Signed-off-by: Egbert Eich <eich@suse.de>
---
 src/H5Dint.c                                      |   6 ++++
 src/H5Odtype.c                                    |   9 +++++
 tools/test/h5dump/h5dumpgentest.c                 |   2 +-
 tools/test/h5stat/testfiles/h5stat_filters-F.ddl  |   6 ++--
 tools/test/h5stat/testfiles/h5stat_filters-UD.ddl |   2 +-
 tools/test/h5stat/testfiles/h5stat_filters-d.ddl  |   2 +-
 tools/test/h5stat/testfiles/h5stat_filters-dT.ddl |   2 +-
 tools/test/h5stat/testfiles/h5stat_filters.ddl    |  16 ++++-----
 tools/test/h5stat/testfiles/h5stat_filters.h5     | Bin 46272 -> 46696 bytes
 tools/testfiles/tallfilters.ddl                   |   2 +-
 tools/testfiles/tfilters.h5                       | Bin 46496 -> 46696 bytes
 tools/testfiles/tnbit.ddl                         |   4 +--
 tools/testfiles/treadintfilter.ddl                |  42 +++++++++++-----------
 13 files changed, 54 insertions(+), 39 deletions(-)

diff --git a/src/H5Dint.c b/src/H5Dint.c
index 02154d4f4c..ae62ce5416 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -932,6 +932,12 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id)
     if (NULL == (oh = H5O_pin(oloc)))
         HGOTO_ERROR(H5E_DATASET, H5E_CANTPIN, FAIL, "unable to pin dataset object header")
 
+    /* Check for creating dataset with unusual datatype */
+    if (H5T_is_numeric_with_unusual_unused_bits(type) &&
+            !(H5O_has_chksum(oh) ||
+                (H5F_RFIC_FLAGS(file) & H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS)))
+        HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "creating dataset with unusual datatype, see documentation for H5Pset_relax_file_integrity_checks for details.");
+
     /* Write the dataspace header message */
     if (H5S_append(file, oh, dset->shared->space) < 0)
         HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update dataspace header message")
diff --git a/src/H5Odtype.c b/src/H5Odtype.c
index 101f9520fb..16a12eb95c 100644
--- a/src/H5Odtype.c
+++ b/src/H5Odtype.c
@@ -782,6 +782,15 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags /*in,out*/, const uint8_t **
             HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown datatype class found")
     } /* end switch */
 
+    /* Check for numeric type w/unusual # of unused bits */
+    if (H5T_is_numeric_with_unusual_unused_bits(dt))
+        /* Throw an error if the object header is not checksummed, unless the
+         * H5F_RFIC_UNUSUAL_NUM_UNUSED_NUMERIC_BITS flag is set with
+         * H5Pset_relax_file_integrity_checks() to suppress it.
+         */
+        if (!(*ioflags & H5O_DECODEIO_RFIC_UNUBNT))
+            HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "datatype has unusually large # of unused bits (prec = %zu bits, size = %zu bytes), possibly corrupted file. See documentation for H5Pset_relax_file_integrity_checks for details.", dt->shared->u.atomic.prec, dt->shared->size);
+
 done:
     /* Cleanup on error */
     if (ret_value < 0)
diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c
index 28caedb92f..18b6af5131 100644
--- a/tools/test/h5dump/h5dumpgentest.c
+++ b/tools/test/h5dump/h5dumpgentest.c
@@ -5578,7 +5578,7 @@ gent_filters(void)
     HDassert(ret >= 0);
 
     tid = H5Tcopy(H5T_NATIVE_INT);
-    H5Tset_precision(tid, H5Tget_size(tid) - 1);
+    H5Tset_precision(tid, (H5Tget_size(tid) * 4) + 1);
     ret = make_dset(fid, "nbit", sid, tid, dcpl, buf1);
     HDassert(ret >= 0);
 
diff --git a/tools/test/h5stat/testfiles/h5stat_filters-F.ddl b/tools/test/h5stat/testfiles/h5stat_filters-F.ddl
index d44445bb2e..065d0c9747 100644
--- a/tools/test/h5stat/testfiles/h5stat_filters-F.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_filters-F.ddl
@@ -4,12 +4,12 @@ File space information for file metadata (in bytes):
 	Superblock extension: 0
 	User block: 0
 	Object headers: (total/unused)
-		Groups: 48/8
-		Datasets(exclude compact data): 4136/1344
+		Groups: 40/0
+		Datasets(exclude compact data): 4128/1088
 		Datatypes: 80/0
 	Groups:
 		B-tree/List: 1200
-		Heap: 288
+		Heap: 384
 	Attributes:
 		B-tree/List: 0
 		Heap: 0
diff --git a/tools/test/h5stat/testfiles/h5stat_filters-UD.ddl b/tools/test/h5stat/testfiles/h5stat_filters-UD.ddl
index 4efafd13c4..9f6335aaed 100644
--- a/tools/test/h5stat/testfiles/h5stat_filters-UD.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_filters-UD.ddl
@@ -1,5 +1,5 @@
 Filename: h5stat_filters.h5
 File space information for datasets' metadata (in bytes):
-	Object headers (total/unused): 4136/1344
+	Object headers (total/unused): 4128/1088
 	Index for Chunked datasets: 31392
 	Heap: 72
diff --git a/tools/test/h5stat/testfiles/h5stat_filters-d.ddl b/tools/test/h5stat/testfiles/h5stat_filters-d.ddl
index 6e6dd6140d..eee7e1845d 100644
--- a/tools/test/h5stat/testfiles/h5stat_filters-d.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_filters-d.ddl
@@ -12,7 +12,7 @@ Dataset dimension information:
 		# of datasets with dimension size 100 - 999: 1
 		Total # of datasets: 1
 Dataset storage information:
-	Total raw data size: 8659
+	Total raw data size: 9046
 	Total external raw data size: 400
 Dataset layout information:
 	Dataset layout counts[COMPACT]: 1
diff --git a/tools/test/h5stat/testfiles/h5stat_filters-dT.ddl b/tools/test/h5stat/testfiles/h5stat_filters-dT.ddl
index b14ca9f974..e513b3a5f2 100644
--- a/tools/test/h5stat/testfiles/h5stat_filters-dT.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_filters-dT.ddl
@@ -12,7 +12,7 @@ Dataset dimension information:
 		# of datasets with dimension size 100 - 999: 1
 		Total # of datasets: 1
 Dataset storage information:
-	Total raw data size: 8659
+	Total raw data size: 9046
 	Total external raw data size: 400
 Dataset layout information:
 	Dataset layout counts[COMPACT]: 1
diff --git a/tools/test/h5stat/testfiles/h5stat_filters.ddl b/tools/test/h5stat/testfiles/h5stat_filters.ddl
index 9f9e146f08..7383f0b31f 100644
--- a/tools/test/h5stat/testfiles/h5stat_filters.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_filters.ddl
@@ -12,12 +12,12 @@ File space information for file metadata (in bytes):
 	Superblock extension: 0
 	User block: 0
 	Object headers: (total/unused)
-		Groups: 48/8
-		Datasets(exclude compact data): 4136/1344
+		Groups: 40/0
+		Datasets(exclude compact data): 4128/1088
 		Datatypes: 80/0
 	Groups:
 		B-tree/List: 1200
-		Heap: 288
+		Heap: 384
 	Attributes:
 		B-tree/List: 0
 		Heap: 0
@@ -50,7 +50,7 @@ Dataset dimension information:
 		# of datasets with dimension size 100 - 999: 1
 		Total # of datasets: 1
 Dataset storage information:
-	Total raw data size: 8659
+	Total raw data size: 9046
 	Total external raw data size: 400
 Dataset layout information:
 	Dataset layout counts[COMPACT]: 1
@@ -91,9 +91,9 @@ Free-space section bins:
 File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR
 File space page size: 4096 bytes
 Summary of file space information:
-  File metadata: 37312 bytes
-  Raw data: 8659 bytes
+  File metadata: 37392 bytes
+  Raw data: 9046 bytes
   Amount/Percent of tracked free space: 0 bytes/0.0%
-  Unaccounted space: 301 bytes
-Total space: 46272 bytes
+  Unaccounted space: 258 bytes
+Total space: 46696 bytes
 External raw data: 400 bytes
diff --git a/tools/test/h5stat/testfiles/h5stat_filters.h5 b/tools/test/h5stat/testfiles/h5stat_filters.h5
index 5b5f4bb7a6..23d68a9b72 100644
Binary files a/tools/test/h5stat/testfiles/h5stat_filters.h5 and b/tools/test/h5stat/testfiles/h5stat_filters.h5 differ
diff --git a/tools/testfiles/tallfilters.ddl b/tools/testfiles/tallfilters.ddl
index ff2454cef7..4fa1742158 100644
--- a/tools/testfiles/tallfilters.ddl
+++ b/tools/testfiles/tallfilters.ddl
@@ -4,7 +4,7 @@ DATASET "all" {
    DATASPACE  SIMPLE { ( 20, 10 ) / ( 20, 10 ) }
    STORAGE_LAYOUT {
       CHUNKED ( 10, 5 )
-      SIZE 458 (1.747:1 COMPRESSION)
+      SIZE 457 (1.751:1 COMPRESSION)
    }
    FILTERS {
       PREPROCESSING SHUFFLE
diff --git a/tools/testfiles/tfilters.h5 b/tools/testfiles/tfilters.h5
index 7c33e55aae..23d68a9b72 100644
Binary files a/tools/testfiles/tfilters.h5 and b/tools/testfiles/tfilters.h5 differ
diff --git a/tools/testfiles/tnbit.ddl b/tools/testfiles/tnbit.ddl
index cf2ac1fdb9..4fa21c3c8f 100644
--- a/tools/testfiles/tnbit.ddl
+++ b/tools/testfiles/tnbit.ddl
@@ -1,10 +1,10 @@
 HDF5 "tfilters.h5" {
 DATASET "nbit" {
-   DATATYPE  32-bit little-endian integer 3-bit precision
+   DATATYPE  32-bit little-endian integer 17-bit precision
    DATASPACE  SIMPLE { ( 20, 10 ) / ( 20, 10 ) }
    STORAGE_LAYOUT {
       CHUNKED ( 10, 5 )
-      SIZE 76 (10.526:1 COMPRESSION)
+      SIZE 428 (1.869:1 COMPRESSION)
    }
    FILTERS {
       COMPRESSION NBIT
diff --git a/tools/testfiles/treadintfilter.ddl b/tools/testfiles/treadintfilter.ddl
index fbad3f6736..7a670a2a25 100644
--- a/tools/testfiles/treadintfilter.ddl
+++ b/tools/testfiles/treadintfilter.ddl
@@ -78,29 +78,29 @@ DATASET "fletcher32" {
    }
 }
 DATASET "nbit" {
-   DATATYPE  32-bit little-endian integer 3-bit precision
+   DATATYPE  32-bit little-endian integer 17-bit precision
    DATASPACE  SIMPLE { ( 20, 10 ) / ( 20, 10 ) }
    DATA {
-   (0,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
-   (1,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
-   (2,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
-   (3,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
-   (4,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
-   (5,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
-   (6,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
-   (7,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
-   (8,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
-   (9,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
-   (10,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
-   (11,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
-   (12,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
-   (13,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
-   (14,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
-   (15,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1,
-   (16,0): 0, 1, 2, 3, -4, -3, -2, -1, 0, 1,
-   (17,0): 2, 3, -4, -3, -2, -1, 0, 1, 2, 3,
-   (18,0): -4, -3, -2, -1, 0, 1, 2, 3, -4, -3,
-   (19,0): -2, -1, 0, 1, 2, 3, -4, -3, -2, -1
+   (0,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+   (1,0): 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+   (2,0): 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+   (3,0): 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+   (4,0): 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+   (5,0): 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
+   (6,0): 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+   (7,0): 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+   (8,0): 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+   (9,0): 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
+   (10,0): 100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+   (11,0): 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+   (12,0): 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+   (13,0): 130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+   (14,0): 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+   (15,0): 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+   (16,0): 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
+   (17,0): 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
+   (18,0): 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+   (19,0): 190, 191, 192, 193, 194, 195, 196, 197, 198, 199
    }
 }
 DATASET "scaleoffset" {
