Hwang hangs during decode in Scanner jobs

[DanFu09]

Scanner pipeline converts the video to HSV and then computes a Histogram over the frame. HSV conversion is a Python op, Histogram computation is the op that ships in stdlib with Scanner.

Two movies that Hwang hangs on: https://console.cloud.google.com/storage/browser/esper/movies/bad/?project=visualdb-1046

Backtrace from relevant Scanner thread in a worker (as far as I can tell, all the workers get stuck):

#0  0x00007f9afcd97827 in sched_yield () at ../sysdeps/unix/syscall-template.S:84
#1  0x00007f9aa2834c1d in __gthread_yield () at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:692
#2  std::this_thread::yield () at /usr/include/c++/5/thread:270
#3  hwang::DecoderAutomata::get_frames (this=0x7f998c0054a0, buffer=buffer@entry=0x7f9843e14010 "", num_frames=num_frames@entry=25) at /opt/scanner-base/thirdparty/build/hwang/hwang/decoder_automata.cpp:218
#4  0x00007f9ae6112632 in scanner::internal::PreEvaluateWorker::yield (this=this@entry=0x7f99ba7f3cd0, item_size=item_size@entry=25, output_entry=...) at /opt/scanner/scanner/engine/evaluate_worker.cpp:246
#5  0x00007f9ae60d6e59 in scanner::internal::(anonymous namespace)::pre_evaluate_driver (input_work=..., output_work=..., args=...) at /opt/scanner/scanner/engine/worker.cpp:211
#6  0x00007f9ae60e7544 in std::_Bind_simple<void (*(std::reference_wrapper<scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> > >, std::reference_wrapper<scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> > >, scanner::internal::PreEvaluateWorkerArgs))(scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> >&, scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> >&, scanner::internal::PreEvaluateWorkerArgs)>::_M_invoke<0ul, 1ul, 2ul>(std::_Index_tuple<0ul, 1ul, 2ul>) (this=<optimized out>) at /usr/include/c++/5/functional:1531
#7  std::_Bind_simple<void (*(std::reference_wrapper<scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> > >, std::reference_wrapper<scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> > >, scanner::internal::PreEvaluateWorkerArgs))(scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> >&, scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> >&, scanner::internal::PreEvaluateWorkerArgs)>::operator()() (this=<optimized out>) at /usr/include/c++/5/functional:1520
#8  std::thread::_Impl<std::_Bind_simple<void (*(std::reference_wrapper<scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> > >, std::reference_wrapper<scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> > >, scanner::internal::PreEvaluateWorkerArgs))(scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> >&, scanner::Queue<std::tuple<std::deque<scanner::internal::TaskStream, std::allocator<scanner::internal::TaskStream> >, scanner::internal::EvalWorkEntry> >&, scanner::internal::PreEvaluateWorkerArgs)> >::_M_run() (this=<optimized out>) at /usr/include/c++/5/thread:115
#9  0x00007f9aa7ebbc80 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007f9afd07e6ba in start_thread (arg=0x7f99ba7f4700) at pthread_create.c:333
#11 0x00007f9afcdb441d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Print out of work_entry from frame 4:

(scanner::internal::EvalWorkEntry &) @0x7f99ba7f3d40: {table_id = 4835, job_index = 3, task_index = 0, row_ids = std::vector of length 1, capacity 1 = {std::vector of length 1000, capacity 1000 = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
      11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67,
      68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
      120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165,
      166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199...}}, columns = std::vector of length 1, capacity 1 = {
    std::vector of length 27, capacity 47 = {{buffer = 0x7f997c004eb0 "@b\001|\231\177", size = 527, is_frame = false, index = 0}, {buffer = 0x7f997c003c20 "@b\001|\231\177", size = 306, is_frame = false, index = 0}, {
        buffer = 0x7f997c12f9c0 "\020<", size = 313, is_frame = false, index = 0}, {buffer = 0x7f997c003a10 "\260\371\022|\231\177", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c005110 "\260\371\022|\231\177", size = 345,
        is_frame = false, index = 0}, {buffer = 0x7f997c003910 "", size = 224, is_frame = false, index = 0}, {buffer = 0x7f997c004d80 "", size = 280, is_frame = false, index = 0}, {buffer = 0x7f997c0043e0 "", size = 345,
        is_frame = false, index = 0}, {buffer = 0x7f997c013d90 "\320C", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c004be0 "\n\004\275\002\346\002\022\004\275\002\346\002\032|", size = 405, is_frame = false, index = 0}, {
        buffer = 0x7f997c013b50 "\220K", size = 405, is_frame = false, index = 0}, {buffer = 0x7f997c016060 "@;\001|\231\177", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c013f80 "P`\001|\231\177", size = 600,
        is_frame = false, index = 0}, {buffer = 0x7f997c014f20 "p?\001|\231\177", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c12de20 "\020O\001|\231\177", size = 345, is_frame = false, index = 0}, {
        buffer = 0x7f997c0149d0 "\020\336\022|\231\177", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c0147e0 "\300I\001|\231\177", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c0153f0 "\320G\001|\231\177",
        size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c12d890 "\n\004\227\005\270\005\022\004\227\005\270\005\032d", size = 345, is_frame = false, index = 0}, {buffer = 0x7f997c014410 "p\324\022|\231\177", size = 390,
        is_frame = false, index = 0}, {buffer = 0x7f997c131560 "", size = 274, is_frame = false, index = 0}, {buffer = 0x7f997c0055f0 "P\025\023|\231\177", size = 327, is_frame = false, index = 0}, {buffer = 0x7f997c131870 "\340U",
        size = 385, is_frame = false, index = 0}, {buffer = 0x7f997c132ba0 "\n\006\320\006\361\006\232\a\022\006\320\006\361\006\232\a\032\337\001", size = 575, is_frame = false, index = 0}, {
        buffer = 0x7f997c131720 "\240'\023|\231\177", size = 307, is_frame = false, index = 0}, {buffer = 0x7f997c131a90 "\020\027\023|\231\177", size = 307, is_frame = false, index = 0}, {
        buffer = 0x7f997c004680 "\n\004\334\a\375\a\022\004\334\a\375\a\032d", size = 265, is_frame = false, index = 0}}}, column_handles = std::vector of length 1, capacity 1 = {{type = scanner::proto::CPU, id = 0}},
  inplace_video = std::vector<bool> of length 1, capacity 64 = {1}, column_types = std::vector of length 1, capacity 1 = {scanner::proto::Video}, needs_configure = 248, needs_reset = 175, last_in_io_packet = 148,
  video_encoding_type = std::vector of length 1, capacity 1 = {scanner::proto::VideoDescriptor_VideoCodecType_H264}, first = true, last_in_task = true, frame_sizes = std::vector of length 0, capacity 0,
  compressed = std::vector<bool> of length 0, capacity 0}