fix(exr): Avoid integer overflow for large deep exr slice strides

Fixes 4540

Signed-off-by: Larry Gritz <[email protected]>

src/libOpenImageIO/imageinout_test.cpp

@@ -19,6 +19,7 @@
 #include <OpenImageIO/imagebufalgo.h>
 #include <OpenImageIO/imageio.h>
 #include <OpenImageIO/unittest.h>
+#include <OpenImageIO/deepdata.h>
 
 using namespace OIIO;
 
@@ -530,7 +531,80 @@ test_read_tricky_sizes()
 }
 
 
+#if 1
+int
+main(int argc, char *argv[])
+{
+    using namespace OIIO;
+    int test_xres = 4096;
+    int test_yres = 2048;
+    int naovs     = 200;
+
+    test_xres = 2048;
+    test_yres = 2048;
+    naovs     = 200;
+
+    OIIO::attribute("threads", 1);
+    OIIO::attribute("exr_threads", 1);
+    int test_nchannels = 3;
+    std::vector<TypeDesc> test_chantypes { TypeHalf, TypeFloat, TypeFloat };
+    std::vector<std::string> test_channames { "A", "Z", "Zback" };
+
+    test_nchannels += naovs * 3;
+    for (int i = 0; i < naovs; ++i) {
+        test_chantypes.push_back(TypeHalf);
+        test_chantypes.push_back(TypeHalf);
+        test_chantypes.push_back(TypeHalf);
+        test_channames.push_back(Strutil::format("data{}.r", i));
+        test_channames.push_back(Strutil::format("data{}.g", i));
+        test_channames.push_back(Strutil::format("data{}.b", i));
+    }
+
+    // Make a deep test image and print info about it (tests DeepData)
+    DeepData dd;
+    dd.init(test_xres, test_nchannels, test_chantypes, test_channames);
+    for (int64_t p = 0; p < dd.pixels(); ++p)
+        dd.set_samples(p, 1);
+    for (int64_t p = 0; p < dd.pixels(); ++p) {
+        int ns = dd.samples(p);
+        for (int s = 0; s < ns; ++s) {
+            // pixels alternate R, G, B. Increasing alpha with each sample
+            // in the pixel, with the last being opaque. Z increases with
+            // each sample (staring at 10.0, + 1.0 for each sample), Zback
+            // is 0.5 past Z.
+            float alpha = float(s + 1) / ns;
+            float r     = (p % 3) == 0 ? 1.0f : 0.0f;
+            float g     = (p % 3) == 1 ? 1.0f : 0.0f;
+            float b     = (p % 3) == 2 ? 1.0f : 0.0f;
+            dd.set_deep_value(p, 0, s, alpha);             // A
+            dd.set_deep_value(p, 1, s, 10.0f + s);         // Z
+            dd.set_deep_value(p, 2, s, 10.0f + s + 0.5f);  // Zback
+
+            for (int i = 0; i < naovs; ++i) {
+                dd.set_deep_value(p, (i * 3) + 3, s, alpha * r);  // R
+                dd.set_deep_value(p, (i * 3) + 4, s, alpha * g);  // G
+                dd.set_deep_value(p, (i * 3) + 5, s, alpha * b);  // B
+            }
+        }
+    }
+
+    // Try to write the test image to an exr file
+    OIIO::print("\nWriting image...\n");
+    ImageSpec spec(test_xres, test_yres, test_nchannels, TypeFloat);
+    spec.channelnames   = test_channames;
+    spec.channelformats = test_chantypes;
+    spec.deep           = true;
+    auto output         = ImageOutput::create("deeptest.exr");
+    output->open("deeptest.exr", spec);
+    for (int i = 0; i < test_yres; ++i) {
+        output->write_deep_scanlines(i, i + 1, 0, dd);
+        Strutil::sync::print("{}\n", i);
+    }
+    output->close();
 
+    print("\nDone.\n");
+}
+#else
 int
 main(int argc, char* argv[])
 {
@@ -552,3 +626,4 @@ main(int argc, char* argv[])
 
     return unit_test_failures;
 }
+#endif

src/openexr.imageio/exrinput.cpp

@@ -1485,7 +1485,7 @@ OpenEXRInput::read_native_deep_scanlines(int subimage, int miplevel, int ybegin,
     try {
         size_t npixels = (yend - ybegin) * m_spec.width;
         chend          = clamp(chend, chbegin + 1, m_spec.nchannels);
-        int nchans     = chend - chbegin;
+        size_t nchans  = chend - chbegin;
 
         // Set up the count and pointers arrays and the Imf framebuffer
         std::vector<TypeDesc> channeltypes;
@@ -1502,16 +1502,17 @@ OpenEXRInput::read_native_deep_scanlines(int subimage, int miplevel, int ybegin,
                               sizeof(unsigned int),
                               sizeof(unsigned int) * m_spec.width);
         frameBuffer.insertSampleCountSlice(countslice);
+        size_t slchans = m_spec.width * nchans;
+        size_t xstride = sizeof(void*) * nchans;
+        size_t ystride = sizeof(void*) * slchans;
+        size_t samplestride = deepdata.samplesize();
 
         for (int c = chbegin; c < chend; ++c) {
-            Imf::DeepSlice slice(
-                part.pixeltype[c],
-                (char*)(&pointerbuf[0] + (c - chbegin) - m_spec.x * nchans
-                        - ybegin * m_spec.width * nchans),
-                sizeof(void*) * nchans,  // xstride of pointer array
-                sizeof(void*) * nchans
-                    * m_spec.width,      // ystride of pointer array
-                deepdata.samplesize());  // stride of data sample
+            Imf::DeepSlice slice(part.pixeltype[c],
+                                 (char*)(&pointerbuf[0] + (c - chbegin)
+                                         - m_spec.x * nchans
+                                         - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_scanline_input_part->setFrameBuffer(frameBuffer);
@@ -1564,7 +1565,7 @@ OpenEXRInput::read_native_deep_tiles(int subimage, int miplevel, int xbegin,
         size_t height  = yend - ybegin;
         size_t npixels = width * height;
         chend          = clamp(chend, chbegin + 1, m_spec.nchannels);
-        int nchans     = chend - chbegin;
+        size_t nchans  = chend - chbegin;
 
         // Set up the count and pointers arrays and the Imf framebuffer
         std::vector<TypeDesc> channeltypes;
@@ -1579,14 +1580,15 @@ OpenEXRInput::read_native_deep_tiles(int subimage, int miplevel, int xbegin,
             Imf::UINT, (char*)(&all_samples[0] - xbegin - ybegin * width),
             sizeof(unsigned int), sizeof(unsigned int) * width);
         frameBuffer.insertSampleCountSlice(countslice);
+        size_t slchans = width * nchans;
+        size_t xstride = sizeof(void*) * nchans;
+        size_t ystride = sizeof(void*) * slchans;
+        size_t samplestride = deepdata.samplesize();
         for (int c = chbegin; c < chend; ++c) {
-            Imf::DeepSlice slice(
-                part.pixeltype[c],
-                (char*)(&pointerbuf[0] + (c - chbegin) - xbegin * nchans
-                        - ybegin * width * nchans),
-                sizeof(void*) * nchans,          // xstride of pointer array
-                sizeof(void*) * nchans * width,  // ystride of pointer array
-                deepdata.samplesize());          // stride of data sample
+            Imf::DeepSlice slice(part.pixeltype[c],
+                                 (char*)(&pointerbuf[0] + (c - chbegin)
+                                         - xbegin * nchans - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_tiled_input_part->setFrameBuffer(frameBuffer);

src/openexr.imageio/exroutput.cpp

@@ -1729,15 +1729,15 @@ OpenEXROutput::write_deep_scanlines(int ybegin, int yend, int /*z*/,
         frameBuffer.insertSampleCountSlice(countslice);
         std::vector<void*> pointerbuf;
         dd->get_pointers(pointerbuf);
+        size_t slchans      = size_t(m_spec.width) * size_t(nchans);
+        size_t xstride      = sizeof(void*) * size_t(nchans);
+        size_t ystride      = sizeof(void*) * slchans;
+        size_t samplestride = dd->samplesize();
         for (int c = 0; c < nchans; ++c) {
             Imf::DeepSlice slice(m_pixeltype[c],
                                  (char*)(&pointerbuf[c] - m_spec.x * nchans
-                                         - ybegin * m_spec.width * nchans),
-                                 sizeof(void*)
-                                     * nchans,  // xstride of pointer array
-                                 sizeof(void*) * nchans
-                                     * m_spec.width,  // ystride of pointer array
-                                 dd->samplesize());   // stride of data sample
+                                         - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_scanline_output_part->setFrameBuffer(frameBuffer);
@@ -1772,17 +1772,17 @@ OpenEXROutput::write_deep_tiles(int xbegin, int xend, int ybegin, int yend,
         return false;
     }
 
-    int nchans         = m_spec.nchannels;
+    size_t nchans      = size_t(m_spec.nchannels);
     const DeepData* dd = &deepdata;
     std::unique_ptr<DeepData> dd_local;  // In case we need a copy
     bool same_chantypes = true;
-    for (int c = 0; c < nchans; ++c)
+    for (size_t c = 0; c < nchans; ++c)
         same_chantypes &= (m_spec.channelformat(c) == deepdata.channeltype(c));
     if (!same_chantypes) {
         // If the channel types don't match, we need to make a copy of the
         // DeepData and convert the channels to the spec's channel types.
         std::vector<TypeDesc> chantypes;
-        if (m_spec.channelformats.size() == size_t(nchans))
+        if (m_spec.channelformats.size() == nchans)
             chantypes = m_spec.channelformats;
         else
             chantypes.resize(nchans, m_spec.format);
@@ -1803,15 +1803,15 @@ OpenEXROutput::write_deep_tiles(int xbegin, int xend, int ybegin, int yend,
         frameBuffer.insertSampleCountSlice(countslice);
         std::vector<void*> pointerbuf;
         dd->get_pointers(pointerbuf);
-        for (int c = 0; c < nchans; ++c) {
+        size_t slchans      = width * nchans;
+        size_t xstride      = sizeof(void*) * nchans;
+        size_t ystride      = sizeof(void*) * slchans;
+        size_t samplestride = dd->samplesize();
+        for (size_t c = 0; c < nchans; ++c) {
             Imf::DeepSlice slice(m_pixeltype[c],
                                  (char*)(&pointerbuf[c] - xbegin * nchans
-                                         - ybegin * width * nchans),
-                                 sizeof(void*)
-                                     * nchans,  // xstride of pointer array
-                                 sizeof(void*) * nchans
-                                     * width,        // ystride of pointer array
-                                 dd->samplesize());  // stride of data sample
+                                         - ybegin * slchans),
+                                 xstride, ystride, samplestride);
             frameBuffer.insert(m_spec.channelnames[c].c_str(), slice);
         }
         m_deep_tiled_output_part->setFrameBuffer(frameBuffer);