-
Notifications
You must be signed in to change notification settings - Fork 24
/
hdf_helper.rg
1437 lines (1323 loc) · 56.1 KB
/
hdf_helper.rg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
-- Generate code for dumping/loading a subset of fields to/from an HDF file.
-- NOTE:
-- * Both functions require an intermediate region to perform the data
-- transfer. This region 's' must have the same size as 'r', and must be
-- partitioned in the same way.
-- * The dimensions will be flipped in the output file.
-- * You need to link to the HDF library to use these functions.
import 'regent'
-------------------------------------------------------------------------------
-- MODULE PARAMETERS
-------------------------------------------------------------------------------
return function(indexType, -- regentlib.index_type
colorType, -- regentlib.index_type
fSpace, -- terralib.struct
flds, -- string*
attrs, -- map(string,terralib.type)
StringAttrs -- map(string,{int, int})
)
local MODULE = {}
MODULE.read = {}
MODULE.write = {}
local VERSION = "1.0.0"
-------------------------------------------------------------------------------
-- FALLBACK MODE
-------------------------------------------------------------------------------
local USE_HDF = assert(os.getenv('USE_HDF')) ~= '0'
if not USE_HDF then
__demand(__inline)
task MODULE.dump(_ : int,
colors : ispace(colorType),
dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
s : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, colors),
p_s : partition(disjoint, s, colors))
where reads(r.[flds]), reads writes(s.[flds]), r * s do
regentlib.assert(false, 'Recompile with USE_HDF=1')
return _
end
__demand(__inline)
task MODULE.load(colors : ispace(colorType),
dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
s : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, colors),
p_s : partition(disjoint, s, colors))
where reads writes(r.[flds]), reads writes(s.[flds]), r * s do
regentlib.assert(false, 'Recompile with USE_HDF=1')
end
for aName,aType in pairs(attrs) do
local __demand(__inline)
task writeAttr(_ : int,
dirname : regentlib.string,
aVal : aType)
regentlib.assert(false, 'Recompile with USE_HDF=1')
return _
end
MODULE.write[aName] = writeAttr
local __demand(__inline)
task readAttr(colors : ispace(colorType),
dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, colors))
regentlib.assert(false, 'Recompile with USE_HDF=1')
return [aType](0)
end
MODULE.read[aName] = readAttr
end
for aName,aPar in pairs(StringAttrs) do
local aNum = aPar[1]
local __demand(__inline)
task writeAttr(_ : int,
colors : ispace(colorType),
dirname : regentlib.string,
Strings : regentlib.string[aNum])
regentlib.assert(false, 'Recompile with USE_HDF=1')
return _
end
MODULE.write[aName] = writeAttr
end
return MODULE
end
-------------------------------------------------------------------------------
-- IMPORTS
-------------------------------------------------------------------------------
local C = regentlib.c
local HDF5 = terralib.includec(assert(os.getenv('HDF_HEADER')))
local UTIL = require 'util'
-- Mark HDF5 tasks as replicable
-- Note: HDF5 must be compiled in thread-safe mode
HDF5.H5Fopen.replicable = true
HDF5.H5Fclose.replicable = true
HDF5.H5Acreate2.replicable = true
HDF5.H5Aexists_by_name.replicable = true
HDF5.H5Aopen.replicable = true
HDF5.H5Aread.replicable = true
HDF5.H5Aclose.replicable = true
HDF5.H5Screate_simple.replicable = true
HDF5.H5Screate.replicable = true
HDF5.H5Sselect_elements.replicable = true
HDF5.H5Sclose.replicable = true
HDF5.H5Tcreate.replicable = true
HDF5.H5Tinsert.replicable = true
HDF5.H5Tcopy.replicable = true
HDF5.H5Tclose.replicable = true
HDF5.H5Dopen2.replicable = true
HDF5.H5Dget_space.replicable = true
HDF5.H5Dread.replicable = true
HDF5.H5Dclose.replicable = true
-------------------------------------------------------------------------------
-- CONSTANTS
-------------------------------------------------------------------------------
-- HACK: Hardcoding missing #define's
HDF5.H5F_ACC_RDONLY = 0
HDF5.H5F_ACC_RDWR = 1
HDF5.H5F_ACC_TRUNC = 2
HDF5.H5P_DEFAULT = 0
HDF5.H5P_DATASET_CREATE = HDF5.H5P_CLS_DATASET_CREATE_ID_g
-------------------------------------------------------------------------------
-- MODULE-LOCAL TASKS
-------------------------------------------------------------------------------
-- string, string? -> terralib.quote
local function err(action, fld)
if fld then
return quote
var stderr = C.fdopen(2, 'w')
C.fprintf(stderr, 'HDF5: Cannot %s for field %s\n', action, fld)
C.fflush(stderr)
C.exit(1)
end
else
return quote
var stderr = C.fdopen(2, 'w')
C.fprintf(stderr, 'HDF5: Cannot %s\n', action)
C.fflush(stderr)
C.exit(1)
end
end
end
-- bool, string, string? -> regentlib.rquote
local function HDF5Assert(condition, action, fld)
local msg
if fld then
msg = "HDF5: Cannot ".. action .." for field " .. fld
else
msg = "HDF5: Cannot ".. action
end
return rquote
regentlib.assert([condition], [msg])
end
end
-- terralib.type -> terralib.expr
local function toPrimHType(T)
return
-- HACK: Hardcoding missing #define's
(T == int) and HDF5.H5T_STD_I32LE_g or
(T == int8) and HDF5.H5T_STD_I8LE_g or
(T == int16) and HDF5.H5T_STD_I16LE_g or
(T == int32) and HDF5.H5T_STD_I32LE_g or
(T == int64) and HDF5.H5T_STD_I64LE_g or
(T == uint) and HDF5.H5T_STD_U32LE_g or
(T == uint8) and HDF5.H5T_STD_U8LE_g or
(T == uint16) and HDF5.H5T_STD_U16LE_g or
(T == uint32) and HDF5.H5T_STD_U32LE_g or
(T == uint64) and HDF5.H5T_STD_U64LE_g or
(T == bool) and HDF5.H5T_STD_U8LE_g or
(T == float) and HDF5.H5T_IEEE_F32LE_g or
(T == double) and HDF5.H5T_IEEE_F64LE_g or
assert(false)
end
local terra create(fname : &int8, size : indexType)
var fid = HDF5.H5Fcreate(fname, HDF5.H5F_ACC_TRUNC,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
if fid < 0 then [err('create file')] end
var dataSpace : HDF5.hid_t
escape
if indexType == int1d then
emit quote
var sizes : HDF5.hsize_t[1]
sizes[0] = size.__ptr
dataSpace = HDF5.H5Screate_simple(1, sizes, [&uint64](0))
if dataSpace < 0 then [err('create 1d dataspace')] end
end
elseif indexType == int2d then
emit quote
-- Legion defaults to column-major layout, so we have to reverse.
var sizes : HDF5.hsize_t[2]
sizes[1] = size.__ptr.x
sizes[0] = size.__ptr.y
dataSpace = HDF5.H5Screate_simple(2, sizes, [&uint64](0))
if dataSpace < 0 then [err('create 2d dataspace')] end
end
elseif indexType == int3d then
emit quote
-- Legion defaults to column-major layout, so we have to reverse.
var sizes : HDF5.hsize_t[3]
sizes[2] = size.__ptr.x
sizes[1] = size.__ptr.y
sizes[0] = size.__ptr.z
dataSpace = HDF5.H5Screate_simple(3, sizes, [&uint64](0))
if dataSpace < 0 then [err('create 3d dataspace')] end
end
else assert(false) end
local header = terralib.newlist() -- terralib.quote*
local footer = terralib.newlist() -- terralib.quote*
-- terralib.type -> terralib.expr
local function toHType(T)
-- TODO: Not supporting: pointers, vectors, non-primitive arrays
if T:isprimitive() then
return toPrimHType(T)
elseif T:isarray() then
local elemType = toHType(T.type)
local arrayType = symbol(HDF5.hid_t, 'arrayType')
header:insert(quote
var dims : HDF5.hsize_t[1]
dims[0] = T.N
var elemType = [elemType]
var [arrayType] = HDF5.H5Tarray_create2(elemType, 1, dims)
if arrayType < 0 then [err('create array type')] end
end)
footer:insert(quote
HDF5.H5Tclose(arrayType)
end)
return arrayType
else assert(false) end
end
-- terralib.struct, set(string), string -> ()
local function emitFieldDecls(fs, whitelist, prefix)
-- TODO: Only supporting pure structs, not fspaces
assert(fs:isstruct())
for _,e in ipairs(fs.entries) do
local name, type = UTIL.parseStructEntry(e)
if whitelist and not whitelist[name] then
-- do nothing
elseif type == int2d then
-- Hardcode special case: int2d structs are stored packed
local hName = prefix..name
local int2dType = symbol(HDF5.hid_t, 'int2dType')
local dataSet = symbol(HDF5.hid_t, 'dataSet')
header:insert(quote
var [int2dType] = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 16)
if int2dType < 0 then [err('create 2d array type', name)] end
var x = HDF5.H5Tinsert(int2dType, "x", 0, HDF5.H5T_STD_I64LE_g)
if x < 0 then [err('add x to 2d array type', name)] end
var y = HDF5.H5Tinsert(int2dType, "y", 8, HDF5.H5T_STD_I64LE_g)
if y < 0 then [err('add y to 2d array type', name)] end
var [dataSet] = HDF5.H5Dcreate2(fid, hName, int2dType, dataSpace,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
if dataSet < 0 then [err('register 2d array type', name)] end
end)
footer:insert(quote
HDF5.H5Dclose(dataSet)
HDF5.H5Tclose(int2dType)
end)
elseif type == int3d then
-- Hardcode special case: int3d structs are stored packed
local hName = prefix..name
local int3dType = symbol(HDF5.hid_t, 'int3dType')
local dataSet = symbol(HDF5.hid_t, 'dataSet')
header:insert(quote
var [int3dType] = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 24)
if int3dType < 0 then [err('create 3d array type', name)] end
var x = HDF5.H5Tinsert(int3dType, "x", 0, HDF5.H5T_STD_I64LE_g)
if x < 0 then [err('add x to 3d array type', name)] end
var y = HDF5.H5Tinsert(int3dType, "y", 8, HDF5.H5T_STD_I64LE_g)
if y < 0 then [err('add y to 3d array type', name)] end
var z = HDF5.H5Tinsert(int3dType, "z", 16, HDF5.H5T_STD_I64LE_g)
if z < 0 then [err('add z to 3d array type', name)] end
var [dataSet] = HDF5.H5Dcreate2(fid, hName, int3dType, dataSpace,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
if dataSet < 0 then [err('register 3d array type', name)] end
end)
footer:insert(quote
HDF5.H5Dclose(dataSet)
HDF5.H5Tclose(int3dType)
end)
elseif type:isstruct() then
emitFieldDecls(type, nil, prefix..name..'.')
else
local hName = prefix..name
local hType = toHType(type)
local dataSet = symbol(HDF5.hid_t, 'dataSet')
header:insert(quote
var hType = [hType]
var [dataSet] = HDF5.H5Dcreate2(
fid, hName, hType, dataSpace,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
if dataSet < 0 then [err('register type', name)] end
end)
footer:insert(quote
HDF5.H5Dclose(dataSet)
end)
end
end
end
emitFieldDecls(fSpace, flds:toSet(), '')
emit quote [header] end
emit quote [footer:reverse()] end
end
HDF5.H5Sclose(dataSpace)
HDF5.H5Fclose(fid)
end
local tileFilename
if indexType == int1d then
__demand(__inline)
task tileFilename(dirname : regentlib.string, bounds : rect1d)
var filename = [&int8](C.malloc(256))
var lo = bounds.lo
var hi = bounds.hi
C.snprintf(filename, 256,
'%s/%ld-%ld.hdf', dirname,
lo, hi)
return filename
end
elseif indexType == int2d then
__demand(__inline)
task tileFilename(dirname : regentlib.string, bounds : rect2d)
var filename = [&int8](C.malloc(256))
var lo = bounds.lo
var hi = bounds.hi
C.snprintf(filename, 256,
'%s/%ld,%ld-%ld,%ld.hdf', dirname,
lo.x, lo.y, hi.x, hi.y)
return filename
end
elseif indexType == int3d then
__demand(__inline)
task tileFilename(dirname : regentlib.string, bounds : rect3d)
var filename = [&int8](C.malloc(256))
var lo = bounds.lo
var hi = bounds.hi
C.snprintf(filename, 256,
'%s/%ld,%ld,%ld-%ld,%ld,%ld.hdf', dirname,
lo.x, lo.y, lo.z, hi.x, hi.y, hi.z)
return filename
end
else assert(false) end
local firstColor =
colorType == int1d and rexpr 0 end or
colorType == int2d and rexpr {0,0} end or
colorType == int3d and rexpr {0,0,0} end or
assert(false)
local oneColor =
colorType == int1d and rexpr 1 end or
colorType == int2d and rexpr {1,1} end or
colorType == int3d and rexpr {1,1,1} end or
assert(false)
local one =
indexType == int1d and rexpr 1 end or
indexType == int2d and rexpr {1,1} end or
indexType == int3d and rexpr {1,1,1} end or
assert(false)
local __demand(__inline)
task createVirtualDataset(fid : HDF5.hid_t,
r : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, ispace(colorType)))
var dataSpace : HDF5.hid_t
rescape
if indexType == int1d then
remit rquote
var sizes : HDF5.hsize_t[1]
sizes[0] = r.bounds.hi - r.bounds.lo + 1
dataSpace = HDF5.H5Screate_simple(1, sizes, [&uint64](0));
[HDF5Assert(rexpr dataSpace >= 0 end, "create 1d dataspace")];
end
elseif indexType == int2d then
remit rquote
-- Legion defaults to column-major layout, so we have to reverse.
var sizes : HDF5.hsize_t[2]
sizes[1] = r.bounds.hi.x - r.bounds.lo.x + 1
sizes[0] = r.bounds.hi.y - r.bounds.lo.y + 1
dataSpace = HDF5.H5Screate_simple(2, sizes, [&uint64](0));
[HDF5Assert(rexpr dataSpace >= 0 end, "create 2d dataspace")];
end
elseif indexType == int3d then
remit rquote
-- Legion defaults to column-major layout, so we have to reverse.
var sizes : HDF5.hsize_t[3]
sizes[2] = r.bounds.hi.x - r.bounds.lo.x + 1
sizes[1] = r.bounds.hi.y - r.bounds.lo.y + 1
sizes[0] = r.bounds.hi.z - r.bounds.lo.z + 1
dataSpace = HDF5.H5Screate_simple(3, sizes, [&uint64](0));
[HDF5Assert(rexpr dataSpace >= 0 end, "create 3d dataspace")];
end
else assert(false) end
local function setVirtualSources(dcpl, name)
if indexType == int1d then
return rquote
var[dcpl] = HDF5.H5Pcreate(HDF5.H5P_DATASET_CREATE);
[HDF5Assert(rexpr dcpl >= 0 end, "create 1d dataspace property list")];
for c in p_r.colors do
var b = p_r[c].bounds
var start : HDF5.hsize_t[1]
start[0] = b.lo - r.bounds.lo
var stride : HDF5.hsize_t[1]
stride[0] = 1
var count : HDF5.hsize_t[1]
count[0] = 1
var block : HDF5.hsize_t[1]
block[0] = b.hi - b.lo + 1
var err = HDF5.H5Sselect_hyperslab(dataSpace, HDF5.H5S_SELECT_SET,
start, stride, count, block);
[HDF5Assert(rexpr err >= 0 end, "select hyperslab in 1d dataspace")];
var src_dataSpace = HDF5.H5Screate_simple(1, block, [&uint64](0));
[HDF5Assert(rexpr dataSpace >= 0 end, "create 1d source dataspace")];
var filename = tileFilename("./", b)
err = HDF5.H5Pset_virtual(dcpl, dataSpace, filename, name, src_dataSpace);
[HDF5Assert(rexpr err >= 0 end, "set virtual dataset", name)];
HDF5.H5Sclose(src_dataSpace)
end
end
elseif indexType == int2d then
return rquote
var [dcpl] = HDF5.H5Pcreate(HDF5.H5P_DATASET_CREATE);
[HDF5Assert(rexpr dcpl >= 0 end, "create 2d dataspace property list")];
for c in p_r.colors do
var b = p_r[c].bounds
var start : HDF5.hsize_t[2]
start[1] = b.lo.x - r.bounds.lo.x
start[0] = b.lo.y - r.bounds.lo.y
var stride : HDF5.hsize_t[2]
stride[1] = 1
stride[0] = 1
var count : HDF5.hsize_t[2]
count[1] = 1
count[0] = 1
var block : HDF5.hsize_t[2]
block[1] = b.hi.x - b.lo.x + 1
block[0] = b.hi.y - b.lo.y + 1
var err = HDF5.H5Sselect_hyperslab(dataSpace, HDF5.H5S_SELECT_SET,
start, stride, count, block);
[HDF5Assert(rexpr err >= 0 end, "select hyperslab in 2d dataspace")];
var src_dataSpace = HDF5.H5Screate_simple(2, block, [&uint64](0));
[HDF5Assert(rexpr dataSpace >= 0 end, "create 2d source dataspace")];
var filename = tileFilename("./", b)
err = HDF5.H5Pset_virtual(dcpl, dataSpace, filename, name, src_dataSpace);
[HDF5Assert(rexpr err >= 0 end, "set virtual dataset", name)];
HDF5.H5Sclose(src_dataSpace)
end
end
elseif indexType == int3d then
return rquote
var [dcpl] = HDF5.H5Pcreate(HDF5.H5P_DATASET_CREATE);
[HDF5Assert(rexpr dcpl >= 0 end, "create 3d dataspace property list")];
for c in p_r.colors do
var b = p_r[c].bounds
var start : HDF5.hsize_t[3]
start[2] = b.lo.x - r.bounds.lo.x
start[1] = b.lo.y - r.bounds.lo.y
start[0] = b.lo.z - r.bounds.lo.z
var stride : HDF5.hsize_t[3]
stride[2] = 1
stride[1] = 1
stride[0] = 1
var count : HDF5.hsize_t[3]
count[2] = 1
count[1] = 1
count[0] = 1
var block : HDF5.hsize_t[3]
block[2] = b.hi.x - b.lo.x + 1
block[1] = b.hi.y - b.lo.y + 1
block[0] = b.hi.z - b.lo.z + 1
var err = HDF5.H5Sselect_hyperslab(dataSpace, HDF5.H5S_SELECT_SET,
start, stride, count, block);
[HDF5Assert(rexpr err >= 0 end, "select hyperslab in 3d dataspace")];
var src_dataSpace = HDF5.H5Screate_simple(3, block, [&uint64](0));
[HDF5Assert(rexpr dataSpace >= 0 end, "create 3d source dataspace")];
var filename = tileFilename("./", b)
err = HDF5.H5Pset_virtual(dcpl, dataSpace, filename, name, src_dataSpace);
[HDF5Assert(rexpr err >= 0 end, "set virtual dataset", name)];
HDF5.H5Sclose(src_dataSpace)
end
end
else assert(false) end
end
local header = terralib.newlist() -- terralib.quote*
local footer = terralib.newlist() -- terralib.quote*
-- terralib.type -> terralib.expr
local function toHType(T)
-- TODO: Not supporting: pointers, vectors, non-primitive arrays
if T:isprimitive() then
return toPrimHType(T)
elseif T:isarray() then
local elemType = toHType(T.type)
local arrayType = regentlib.newsymbol(HDF5.hid_t, 'arrayType')
local SIZE = T.N
header:insert(rquote
var dims : HDF5.hsize_t[1]
dims[0] = SIZE
var elemType = [elemType]
var [arrayType] = HDF5.H5Tarray_create2(elemType, 1, dims);
[HDF5Assert(rexpr arrayType >= 0 end, "create array type")];
end)
footer:insert(rquote
HDF5.H5Tclose(arrayType)
end)
return arrayType
else assert(false) end
end
-- terralib.struct, set(string), string -> ()
local function emitFieldDecls(fs, whitelist, prefix)
-- TODO: Only supporting pure structs, not fspaces
assert(fs:isstruct())
for _,e in ipairs(fs.entries) do
local name, type = UTIL.parseStructEntry(e)
if whitelist and not whitelist[name] then
-- do nothing
elseif type == int2d then
-- Hardcode special case: int2d structs are stored packed
local hName = prefix..name
local int2dType = regentlib.newsymbol(HDF5.hid_t, 'int2dType')
local dataSet = regentlib.newsymbol(HDF5.hid_t, 'dataSet')
local dcpl = regentlib.newsymbol(HDF5.hid_t, 'dcpl')
header:insert(quote
var [int2dType] = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 16);
[HDF5Assert(rexpr int2dType >= 0 end, "create 2d array type", name)];
var x = HDF5.H5Tinsert(int2dType, "x", 0, HDF5.H5T_STD_I64LE_g);
[HDF5Assert(rexpr x >= 0 end, "add x to 2d array type", name)];
var y = HDF5.H5Tinsert(int2dType, "y", 8, HDF5.H5T_STD_I64LE_g);
[HDF5Assert(rexpr y >= 0 end, "add y to 2d array type", name)];
[setVirtualSources(dcpl, name)];
var [dataSet] = HDF5.H5Dcreate2(fid, hName, int2dType, dataSpace,
HDF5.H5P_DEFAULT, dcpl, HDF5.H5P_DEFAULT);
[HDF5Assert(rexpr dataSet >= 0 end, "register 2d array type", name)];
HDF5.H5Pclose(dcpl)
end)
footer:insert(quote
HDF5.H5Dclose(dataSet)
HDF5.H5Pclose(dcpl)
HDF5.H5Tclose(int2dType)
end)
elseif type == int3d then
-- Hardcode special case: int3d structs are stored packed
local hName = prefix..name
local int3dType = regentlib.newsymbol(HDF5.hid_t, 'int3dType')
local dataSet = regentlib.newsymbol(HDF5.hid_t, 'dataSet')
local dcpl = regentlib.newsymbol(HDF5.hid_t, 'dcpl')
header:insert(quote
var [int3dType] = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 24);
[HDF5Assert(rexpr int3dType >= 0 end, "create 3d array type", name)];
var x = HDF5.H5Tinsert(int3dType, "x", 0, HDF5.H5T_STD_I64LE_g);
[HDF5Assert(rexpr x >= 0 end, "add x to 3d array type", name)];
var y = HDF5.H5Tinsert(int3dType, "y", 8, HDF5.H5T_STD_I64LE_g);
[HDF5Assert(rexpr y >= 0 end, "add y to 3d array type", name)];
var z = HDF5.H5Tinsert(int3dType, "z", 16, HDF5.H5T_STD_I64LE_g);
[HDF5Assert(rexpr z >= 0 end, "add z to 3d array type", name)];
[setVirtualSources(dcpl, name)];
var [dataSet] = HDF5.H5Dcreate2(fid, hName, int3dType, dataSpace,
HDF5.H5P_DEFAULT, dcpl, HDF5.H5P_DEFAULT);
[HDF5Assert(rexpr dataSet >= 0 end, "register 3d array type", name)];
HDF5.H5Pclose(dcpl)
end)
footer:insert(quote
HDF5.H5Dclose(dataSet)
HDF5.H5Pclose(dcpl)
HDF5.H5Tclose(int3dType)
end)
elseif type:isstruct() then
emitFieldDecls(type, nil, prefix..name..'.')
else
local hName = prefix..name
local hType = toHType(type)
local dataSet = regentlib.newsymbol(HDF5.hid_t, 'dataSet')
local dcpl = regentlib.newsymbol(HDF5.hid_t, 'dcpl')
header:insert(rquote
var hType = [hType];
[setVirtualSources(dcpl, name)];
var [dataSet] = HDF5.H5Dcreate2(
fid, hName, hType, dataSpace,
HDF5.H5P_DEFAULT, dcpl, HDF5.H5P_DEFAULT);
[HDF5Assert(rexpr dataSet >= 0 end, "register type", name)];
end)
footer:insert(rquote
HDF5.H5Dclose(dataSet)
HDF5.H5Pclose(dcpl)
end)
end
end
end
emitFieldDecls(fSpace, flds:toSet(), '')
remit rquote [header] end
remit rquote [footer:reverse()] end
end
HDF5.H5Sclose(dataSpace)
end
local function mkH5Types(pointType)
local H5TpointType
local H5TRectType
local pStruct
local point2struct
local bStruct
local rect2struct
if pointType == int1d then
__demand(__inline)
task H5TpointType()
var pointType = HDF5.H5Tcopy(HDF5.H5T_STD_I64LE_g)
regentlib.assert(pointType >= 0, "HDF5: Cannot create 1d point type")
return pointType
end
__demand(__inline)
task H5TRectType(pointType : HDF5.hid_t)
var rectType = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 16)
regentlib.assert(rectType >= 0, "HDF5: Cannot create rect1d type")
var lo = HDF5.H5Tinsert(rectType, "lo", 0, pointType)
regentlib.assert(lo >= 0, "HDF5: Cannot add lo to rect1d type")
var hi = HDF5.H5Tinsert(rectType, "hi", 8, pointType)
regentlib.assert(hi >= 0, "HDF5: Cannot add hi to rect1d type")
return rectType
end
pStruct = int64
__demand(__inline)
task point2struct(p : int1d)
var out : int64 = p
return out
end
struct bStruct {
lo : int64;
hi : int64;
}
__demand(__inline)
task rect2struct(b : rect1d)
var out : bStruct
out.lo = b.lo
out.hi = b.hi
return out
end
elseif pointType == int2d then
__demand(__inline)
task H5TpointType()
var pointType = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 16)
regentlib.assert(pointType >= 0 , "HDF5: Cannot create 2d point type")
var x = HDF5.H5Tinsert(pointType, "x", 0, HDF5.H5T_STD_I64LE_g)
regentlib.assert(x >= 0, "HDF5: Cannot add x to 2d point type")
var y = HDF5.H5Tinsert(pointType, "y", 8, HDF5.H5T_STD_I64LE_g)
regentlib.assert(y >= 0, "HDF5: Cannot add y to 2d point type")
return pointType
end
__demand(__inline)
task H5TRectType(pointType : HDF5.hid_t)
var rectType = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 32)
regentlib.assert(rectType >= 0, "HDF5: Cannot create rect2d type")
var lo = HDF5.H5Tinsert(rectType, "lo", 0, pointType)
regentlib.assert(lo >= 0, "HDF5: Cannot add lo to rect2d type")
var hi = HDF5.H5Tinsert(rectType, "hi", 16, pointType)
regentlib.assert(hi >= 0, "HDF5: Cannot add hi to rect2d type")
return rectType
end
pStruct = int64[2]
__demand(__inline)
task point2struct(p : int2d)
var out : int64[2]
out[0] = p.x
out[1] = p.y
return out
end
struct bStruct {
lo : int64[2];
hi : int64[2];
}
__demand(__inline)
task rect2struct(b : rect2d)
var out : bStruct
out.lo[0] = b.lo.x
out.lo[1] = b.lo.y
out.hi[0] = b.hi.x
out.hi[1] = b.hi.y
return out
end
elseif pointType == int3d then
__demand(__inline)
task H5TpointType()
var pointType = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 24)
regentlib.assert(pointType >= 0, "HDF5: Cannot create 3d point type")
var x = HDF5.H5Tinsert(pointType, "x", 0, HDF5.H5T_STD_I64LE_g)
regentlib.assert(x >= 0, "HDF5: Cannot add x to 3d point type")
var y = HDF5.H5Tinsert(pointType, "y", 8, HDF5.H5T_STD_I64LE_g)
regentlib.assert(y >= 0, "HDF5: Cannot add y to 3d point type")
var z = HDF5.H5Tinsert(pointType, "z", 16, HDF5.H5T_STD_I64LE_g)
regentlib.assert(z >= 0, "HDF5: Cannot add z to 3d point type")
return pointType
end
__demand(__inline)
task H5TRectType(pointType : HDF5.hid_t)
var rectType = HDF5.H5Tcreate(HDF5.H5T_COMPOUND, 48)
regentlib.assert(rectType >= 0, "HDF5: Cannot create rect3d type")
var lo = HDF5.H5Tinsert(rectType, "lo", 0, pointType)
regentlib.assert(lo >= 0, "HDF5: Cannot add lo to rect3d type")
var hi = HDF5.H5Tinsert(rectType, "hi", 24, pointType)
regentlib.assert(hi >= 0, "HDF5: Cannot add hi to rect3d type")
return rectType
end
pStruct = int64[3]
__demand(__inline)
task point2struct(p : int3d)
var out : int64[3]
out[0] = p.x
out[1] = p.y
out[2] = p.z
return out
end
struct bStruct {
lo : int64[3];
hi : int64[3];
}
__demand(__inline)
task rect2struct(b : rect3d)
var out : bStruct
out.lo[0] = b.lo.x
out.lo[1] = b.lo.y
out.lo[2] = b.lo.z
out.hi[0] = b.hi.x
out.hi[1] = b.hi.y
out.hi[2] = b.hi.z
return out
end
else assert(false) end
return H5TpointType, H5TRectType, pStruct, point2struct, bStruct, rect2struct
end
local H5TindexType, H5TindexRectType, indexStruct, index2struct, IndexRectStruct, indexRect2struct = mkH5Types(indexType)
local H5TcolorType, H5TcolorRectType, colorStruct, color2struct, ColorRectStruct, colorRect2struct = mkH5Types(colorType)
local H5TPartitionSpace
local H5TselectPartitionPoint
if colorType == int1d then
__demand(__inline)
task H5TPartitionSpace(colors : ispace(colorType))
var aSize : HDF5.hsize_t[1]
aSize[0] = colors.bounds.hi - colors.bounds.lo + 1
var sid = HDF5.H5Screate_simple(1, aSize, aSize)
regentlib.assert(sid >= 0, "HDF5: Cannot create partitions attribute dataspace")
return sid
end
__demand(__inline)
task H5TselectPartitionPoint(c : colorType, sid : HDF5.hid_t)
var aSize : HDF5.hsize_t[1]
aSize[0] = c
var res = HDF5.H5Sselect_elements(sid, HDF5.H5S_SELECT_SET, 1, aSize)
regentlib.assert(res >= 0, "HDF5: Cannot select partition point in dataspace")
end
elseif colorType == int2d then
__demand(__inline)
task H5TPartitionSpace(colors : ispace(colorType))
var aSize : HDF5.hsize_t[2]
aSize[0] = colors.bounds.hi.y - colors.bounds.lo.y + 1
aSize[1] = colors.bounds.hi.x - colors.bounds.lo.x + 1
var sid = HDF5.H5Screate_simple(2, aSize, aSize)
regentlib.assert(sid >= 0, "HDF5: Cannot create partitions attribute dataspace")
return sid
end
__demand(__inline)
task H5TselectPartitionPoint(c : colorType, sid : HDF5.hid_t)
var aSize : HDF5.hsize_t[2]
aSize[0] = c.y
aSize[1] = c.x
var res = HDF5.H5Sselect_elements(sid, HDF5.H5S_SELECT_SET, 1, aSize)
regentlib.assert(res >= 0, "HDF5: Cannot select partition point in dataspace")
end
elseif colorType == int3d then
__demand(__inline)
task H5TPartitionSpace(colors : ispace(colorType))
var aSize : HDF5.hsize_t[3]
aSize[0] = colors.bounds.hi.z - colors.bounds.lo.z + 1
aSize[1] = colors.bounds.hi.y - colors.bounds.lo.y + 1
aSize[2] = colors.bounds.hi.x - colors.bounds.lo.x + 1
var sid = HDF5.H5Screate_simple(3, aSize, aSize)
regentlib.assert(sid >= 0, "HDF5: Cannot create partitions attribute dataspace")
return sid
end
__demand(__inline)
task H5TselectPartitionPoint(c : colorType, sid : HDF5.hid_t)
var aSize : HDF5.hsize_t[3]
aSize[0] = c.z
aSize[1] = c.y
aSize[2] = c.x
var res = HDF5.H5Sselect_elements(sid, HDF5.H5S_SELECT_SET, 1, aSize)
regentlib.assert(res >= 0, "HDF5: Cannot select partition point in dataspace")
end
else assert(false) end
-------------------------------------------------------------------------------
-- EXPORTED TASKS
-------------------------------------------------------------------------------
local colorRectType =
colorType == int1d and rect1d or
colorType == int2d and rect2d or
colorType == int3d and rect3d or
assert(false)
local __demand(__leaf) -- MANUALLY PARALLELIZED, NO CUDA, NO OPENMP
task dumpMasterFile(_ : int,
dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, ispace(colorType)))
-- Create file
var filename = [&int8](C.malloc(256))
C.snprintf(filename, 256, '%s/master.hdf', [&int8](dirname))
var fid = HDF5.H5Fcreate(filename, HDF5.H5F_ACC_TRUNC,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
regentlib.assert(fid >= 0, "HDF5: Cannot create master file")
-- Create scalar data space
var sid = HDF5.H5Screate(HDF5.H5S_SCALAR)
regentlib.assert(sid >= 0, "HDF5: Cannot create attribute dataspace")
-- Store IO version
var stringType = HDF5.H5Tcopy(HDF5.H5T_C_S1_g)
var res = HDF5.H5Tset_size (stringType, 8)
regentlib.assert(res >= 0, "HDF5: Cannot set attribute size")
var aid = HDF5.H5Acreate2(fid, "IO_VERSION", stringType, sid,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
regentlib.assert(aid >= 0, "HDF5: Cannot create IO_VERSION attribute")
var v : int8[8]
C.snprintf(&v[0], 8, VERSION)
res = HDF5.H5Awrite(aid, stringType, &v)
regentlib.assert(res >= 0, "HDF5: Cannot write IO_VERSION attribute")
HDF5.H5Aclose(aid)
-- Register data types to describe index spaces
var indexType = H5TindexType()
var iRectType = H5TindexRectType(indexType)
var colorType = H5TcolorType()
var cRectType = H5TcolorRectType(colorType)
-- Store main index space
aid = HDF5.H5Acreate2(fid, "baseIndexSpace", iRectType, sid,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
regentlib.assert(aid >= 0, "HDF5: Cannot create baseIndexSpace attribute")
var b = indexRect2struct(r.bounds)
res = HDF5.H5Awrite(aid, iRectType, &b)
regentlib.assert(res >= 0, "HDF5: Cannot write baseIndexSpace attribute")
HDF5.H5Aclose(aid)
-- Store colors
aid = HDF5.H5Acreate2(fid, "tilesNumber", colorType, sid,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
regentlib.assert(aid >= 0, "HDF5: Cannot create tilesNumber attribute")
-- regentlib.assert(colors.bounds.lo == firstColor, 'tiles should have 0 as lower bound')
var colors_hi = color2struct(p_r.colors.bounds.hi - p_r.colors.bounds.lo + oneColor)
res = HDF5.H5Awrite(aid, colorType, &colors_hi)
regentlib.assert(res >= 0, "HDF5: Cannot write tilesNumber attribute")
HDF5.H5Aclose(aid)
-- Store partitions
var part_sid = H5TPartitionSpace(p_r.colors)
var part_aid = HDF5.H5Dcreate2(fid, "partitions", iRectType, part_sid,
HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT, HDF5.H5P_DEFAULT)
regentlib.assert(part_aid >= 0, "HDF5: Cannot create partitions storage")
var part_fid = HDF5.H5Dget_space(part_aid)
regentlib.assert(part_fid >= 0, "HDF5: Cannot create partitions file_id")
for c in p_r.colors do
H5TselectPartitionPoint(c, part_fid)
var b = indexRect2struct(p_r[c].bounds)
var res = HDF5.H5Dwrite(part_aid, iRectType, sid, part_fid,
HDF5.H5P_DEFAULT, &b)
regentlib.assert(res >= 0, "HDF5: Cannot write partition storage")
end
HDF5.H5Sclose(part_fid)
HDF5.H5Dclose(part_aid)
HDF5.H5Sclose(part_sid)
-- Create a virtual dataset that represents the entire index space
createVirtualDataset(fid, r, p_r)
-- Clean up
HDF5.H5Tclose(indexType)
HDF5.H5Tclose(iRectType)
HDF5.H5Tclose(colorType)
HDF5.H5Tclose(cRectType)
HDF5.H5Tclose(stringType)
HDF5.H5Sclose(sid)
HDF5.H5Fclose(fid)
C.free(filename)
return _
end
local __demand(__inner) -- NOT LEAF, MANUALLY PARALLELIZED, NO CUDA, NO OPENMP
task dumpTile(_ : int,
dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
s : region(ispace(indexType), fSpace))
where reads(r.[flds]), reads writes(s.[flds]), r * s do
if (r.volume ~= 0) then
var filename = tileFilename([&int8](dirname), r.bounds)
create(filename, r.bounds.hi - r.bounds.lo + one)
attach(hdf5, s.[flds], filename, regentlib.file_read_write)
acquire(s.[flds])
copy(r.[flds], s.[flds])
release(s.[flds])
detach(hdf5, s.[flds])
C.free(filename)
end
end
__demand(__inline)
task MODULE.dump(_ : int,
colors : ispace(colorType),
dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
s : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, colors),
p_s : partition(disjoint, s, colors))
where reads(r.[flds]), reads writes(s.[flds]), r * s do
-- Dump the actual data
for c in colors do
dumpTile(_, dirname, p_r[c], p_s[c])
end
-- Dump all the domain info into a master file
return dumpMasterFile(_, dirname, r, p_r)
end
local __demand(__inline)
task checkMasterFile(dirname : regentlib.string,
r : region(ispace(indexType), fSpace),
p_r : partition(disjoint, r, ispace(colorType)))
-- This flag is going to tell us if we need to repart our domain
var repart = false
-- Open file
var filename = [&int8](C.malloc(256))
C.snprintf(filename, 256, '%s/master.hdf', [&int8](dirname))
var fid = HDF5.H5Fopen(filename, HDF5.H5F_ACC_RDONLY,
HDF5.H5P_DEFAULT)
if (fid < 0) then
-- We did not find any master file.
-- Assume everything is ok and we do not need to repart our data
-- for backward compatibility
repart = false
end
-- TODO: Check IO version?
-- Register data types to describe index spaces
var H5indexType = H5TindexType()
var H5iRectType = H5TindexRectType(H5indexType)
var H5colorType = H5TcolorType()
if (fid >= 0) then
-- Check that we are working on the same index space
var res = HDF5.H5Aexists_by_name(fid, ".", "baseIndexSpace",
HDF5.H5P_DEFAULT)
regentlib.assert(res >= 1, "HDF5: Cannot find baseIndexSpace attribute")
var aid = HDF5.H5Aopen(fid, "baseIndexSpace", HDF5.H5P_DEFAULT)
regentlib.assert(aid >= 0, "HDF5: Cannot open baseIndexSpace attribute")