Skip to content

Commit

Permalink
Merge pull request #14 from getumen/fix-parallel-comp
Browse files Browse the repository at this point in the history 
✨ parallel compile
  • Loading branch information
@getumen
getumen authored Jul 31, 2022
2 parents 9eea921 + 1b22f76 commit 2e3f41e
Show file tree
Hide file tree
Showing 14 changed files with 4,701 additions and 1,017 deletions.
49 changes: 33 additions & 16 deletions compiler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"os/exec"
"path"
"runtime"
"sync"
)

var (
Expand Down Expand Up @@ -161,26 +162,42 @@ func (c *Compiler) ExportSharedLib(
return errCompile
}

objectNameList := make([]string, 0)
objectNameChan := make(chan string)
wg := sync.WaitGroup{}

for _, source := range recipe.Sources {
wg.Add(1)

objectName := path.Join(dir, fmt.Sprintf("%s.o", source.Name))
go func(sourceName string) {
defer wg.Done()
objectName := path.Join(dir, fmt.Sprintf("%s.o", sourceName))

objOptions := []string{
"-c",
"-O3",
"-o",
objectName,
path.Join(dir, fmt.Sprintf("%s.c", source.Name)),
"-fPIC",
"-std=c99",
}
objOptions = append(objOptions, compileOptions...)
objOptions := []string{
"-Ofast",
"-o",
objectName,
"-c",
path.Join(dir, fmt.Sprintf("%s.c", sourceName)),
"-fPIC",
"-std=c99",
}
objOptions = append(objOptions, compileOptions...)

err = exec.Command("gcc", objOptions...).Run()
if err != nil {
return err
}
_, err := exec.Command("gcc", objOptions...).Output()
if err != nil {
log.Fatalf("fail to compile %s", objectName)
}
objectNameChan <- objectName
}(source.Name)
}

go func() {
wg.Wait()
close(objectNameChan)
}()

objectNameList := make([]string, 0)
for objectName := range objectNameChan {
objectNameList = append(objectNameList, objectName)
}

Expand Down
2 changes: 1 addition & 1 deletion testdata/annotation.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
[[114,67,47,65,2,10,37],[114,67,47,65,2,10,37],[114,67,47,65,2,7,40],[114,67,47,65,2,10,37],[114,67,47,65,2,7,40],[114,75,39,71,4,0,39],[114,67,47,65,2,10,37],[114,67,47,65,2,7,40],[114,75,39,71,4,0,39],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,75,39,71,4,0,39],[114,67,47,65,2,10,37],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,67,47,65,2,7,40],[114,75,39,71,4,0,39],[114,75,39,71,4,0,39],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,67,47,65,2,10,37],[114,75,39,71,4,0,39],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,67,47,65,2,10,37],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,75,39,71,4,0,39],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,67,47,65,2,13,34],[114,75,39,71,4,0,39],[114,67,47,65,2,10,37],[114,75,39,71,4,39,0],[114,67,47,65,2,13,34],[114,75,39,71,4,2,37],[114,67,47,65,2,13,34],[114,75,39,71,4,39,0],[114,67,47,65,2,10,37],[114,67,47,65,2,13,34],[114,75,39,71,4,39,0],[114,67,47,65,2,7,40],[114,75,39,71,4,39,0],[114,67,47,65,2,13,34],[114,75,39,71,4,39,0],[114,67,47,65,2,10,37],[114,75,39,71,4,2,37],[114,67,47,65,2,11,36],[114,75,39,71,4,39,0],[114,67,47,65,2,13,34],[114,75,39,71,4,2,37],[114,67,47,65,2,11,36],[114,75,39,71,4,39,0],[114,77,37,72,5,2,35],[114,67,47,65,2,13,34],[114,75,39,71,4,39,0],[114,67,47,65,2,11,36],[114,75,39,71,4,2,37],[114,75,39,71,4,2,37],[114,67,47,65,2,11,36],[114,77,37,72,5,2,35],[114,75,39,71,4,39,0],[114,67,47,65,2,13,34],[114,75,39,71,4,2,37],[114,67,47,65,2,11,36],[114,75,39,71,4,39,0],[114,77,37,72,5,2,35],[114,67,47,65,2,13,34],[114,75,39,71,4,39,0],[114,67,47,65,2,11,36],[114,75,39,71,4,2,37],[114,67,47,65,2,13,34],[114,75,39,71,4,2,37],[114,75,39,71,4,39,0],[114,67,47,65,2,11,36],[114,77,37,72,5,3,34],[114,77,37,72,5,3,34],[114,75,39,71,4,2,37],[114,67,47,65,2,11,36],[114,77,37,72,5,3,34],[114,75,39,71,4,39,0],[114,77,37,72,5,3,34],[114,67,47,65,2,11,36],[114,77,37,72,5,3,34],[114,75,39,71,4,2,37],[114,67,47,65,2,7,40],[114,77,37,72,5,3,34],[114,75,39,71,4,39,0],[114,67,47,65,2,11,36],[114,77,37,72,5,3,34],[114,75,39,71,4,2,37],[114,77,37,72,5,3,34],[114,67,47,65,2,11,36],[114,75,39,71,4,39,0],[114,77,37,72,5,3,34],[114,63,51,57,6,4,47],[114,67,47,65,2,11,36],[114,76,38,72,4,38,0]]
[[114,67,47,65,2,10,37,64,1,0,2,8,2,37,0,63,1,7,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,37,0,7,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,37,0,0,2,8,0],[114,67,47,65,2,10,37,64,1,0,2,8,2,37,0,8,0],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,2,0,2,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,1,1,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,34,3,63,1,0,2,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,64,0,2,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,0,2,1,36,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,64,0,2,0,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,1,36,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,64,0,2,0,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,1,0,37],[114,67,47,65,2,10,37,64,1,0,2,8,2,34,3,62,2,0,2,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,64,0,2,0,2,1,36,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,62,2,7,1],[114,75,39,71,4,0,39,68,3,1,3,3,36,55,13,1,2,1,2,11,2],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,1,1,36],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,64,0,2,0,2,1,0,37],[114,75,39,71,4,0,39,68,3,1,3,3,36,55,13,1,2,1,2,11,2],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,1,36,1],[114,75,39,71,4,0,39,68,3,1,3,3,36,55,13,1,2,1,2,11,2],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,62,2,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,64,0,0,2,2,1,36,1],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,62,2,7,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,0,2,1,36,1],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,64,0,6,4],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,7,40,64,1,0,2,6,1,3,37,62,2,2,1,1,36],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,10,0],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,64,0,10,0],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,0,2,7,3,1,61],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,6,4,1,61],[114,75,39,71,4,0,39,68,3,1,3,3,36,53,15,1,2,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,7,3,1,61],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,64,0,0,2,7,1,1,63],[114,75,39,71,4,2,37,68,3,1,3,1,1,2,35,53,15,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,10,0,1,61],[114,75,39,71,4,2,37,68,3,1,3,1,1,2,35,53,15,1,2,13,2,12,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,6,4,1,61],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,63,1,7,3,1,62],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,31,1,67],[114,67,47,65,2,10,37,64,1,0,2,8,2,34,3,63,1,7,1,1,62],[114,67,47,65,2,13,34,64,1,0,2,10,3,63,1,2,0,6,4,1,62],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,64,0,6,4,1,63],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,6,4,1,61],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,31,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,7,3,1,61],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,2,33,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,62,2,6,4,1,61],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,31,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,64,0,0,0,6,4,1,63],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,0,35,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,0,34,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,6,4,1,58,3,2],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,59,5,0,0,7,1,1,58,3,2],[114,67,47,65,2,13,34,64,1,0,2,10,3,63,1,0,0,6,4,1,62],[114,75,39,71,4,2,37,70,1,0,4,1,1,2,35,68,2,2,2,3,32,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,0,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,6,4,1,58,3,2],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,0,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,0,1,67],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,6,4,1,58,3,2],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,0,1,67],[114,67,47,65,2,10,37,64,1,0,2,8,2,34,3,59,5,0,0,7,1,1,58,4,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,6,4,1,2,1,58,4,1],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,68,2,2,2,3,0,1,67],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,59,11,2,2,3,0,8,3,1,2],[114,67,47,65,2,10,37,64,1,0,2,8,2,36,1,59,5,0,0,7,1,1,58,3,2],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,6,4,1,2,1,58,4,1],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,59,11,2,2,1,2,8,3,1,2],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,10,0,1,2,1,58,2,3],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,59,11,2,2,1,2,8,3,1,2],[114,67,47,65,2,10,37,64,1,0,2,2,8,34,3,59,5,0,0,4,4,1,58,2,3],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,59,11,2,2,3,0,8,3,1,2],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,10,0,1,2,1,58,2,3],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,59,11,2,2,3,0,8,3,1,2],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,10,0,1,2,1,58,2,3],[114,75,39,71,4,2,37,70,1,0,4,1,1,3,34,59,11,2,2,3,0,8,3,2,1],[114,67,47,65,2,10,37,64,1,0,2,2,8,36,1,59,5,0,0,4,4,1,58,2,3],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,5,5,1,2,1,58,2,3],[114,75,39,71,4,2,37,56,15,1,3,1,1,3,34,11,4,3,0,1,3,2,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,5,5,1,2,1,58,2,3],[114,75,39,71,4,2,37,56,15,1,3,1,1,2,35,11,4,1,3,2,1],[114,67,47,65,2,13,34,64,1,0,2,10,3,59,5,0,0,5,5,1,2,1,58,2,3]]
Binary file modified testdata/compiled_model.so
View file
Binary file not shown.
26 changes: 18 additions & 8 deletions testdata/compiled_model/main.c
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,25 @@
static const int th_len[] = {
0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 5, 3, 3, 2, 0, 0,
0, 4, 0, 0,
4, 8, 0, 0, 4, 3, 2, 3, 0, 2, 3, 3, 1, 6, 1, 0, 3, 2, 0, 3, 7, 9, 4, 2, 4, 1,
5, 8, 2, 3,
};
static const int th_begin[] = {
0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 9, 12, 15, 17,
17, 17, 17, 21, 21,
0, 4, 12, 12, 12, 16, 19, 21, 24, 24, 26, 29, 32, 33, 39, 40, 40, 43, 45, 45,
48, 55, 64, 68, 70, 74, 75, 80, 88, 90,
};
static const float threshold[] = {
0.072140001, 0.051279999, 3.4375, 0.1222, 14.855, 15.385, 16.795, 16.805,
16.83, 19.91, 20.404999, 22.849998, 101.55, 103.25, 114.45, 868.19995,
957.45001, 0.14234999, 0.14559999, 0.14655, 0.15889999,
14.514999, 14.74, 15.445, 15.875, 16.190001, 16.375, 18.68, 19.544998, 20.49,
21.055, 21.575001, 22.455, 0.083779998, 0.108, 0.11625, 0.11965, 0.066914998,
0.073200002, 0.13495, 0.038989998, 0.088660002, 0.025234999, 0.051279999,
0.091004997, 0.060304999, 0.076870002, 0.24635001, 0.24744999, 0.57210004,
0.60924995, 1.244, 1.3905001, 4.1184998, 19.315001, 21.849998, 24.889999,
30.385, 42.949997, 48.700001, 0.0033090001, 0.01929, 0.101765, 0.1103,
0.0093224999, 0.011815, 0.0013145, 0.001315, 0.0095415004, 14.43, 14.855,
15.385, 15.48, 16.805, 16.83, 18.040001, 20.285, 25.654999, 25.924999,
26.334999, 27.43, 30.145, 33.349998, 37.965, 38.43, 101.2, 103.25, 114.45,
115.25, 740, 957.45001, 0.13519999, 0.14125, 0.16605, 0.16749999, 0.43375,
0.2071, 0.22055, 0.23684999, 0.25749999, 0.48014998, 0.078604996, 0.1112,
0.1358, 0.14234999, 0.14365, 0.14655, 0.14895001, 0.15075, 0.31105,
0.35710001, 0.079060003, 0.11765, 0.1222,
};

#include <stdlib.h>
Expand All @@ -31,7 +41,7 @@ static inline int quantize(float val, unsigned fid) {
// It is possible th_begin[i] == [total_num_threshold]. This means that
// all features i, (i+1), ... are not used for any of the splits in the model.
// So in this case, just return something
if (offset == 21 || val < array[0]) {
if (offset == 93 || val < array[0]) {
return -10;
}
while (low + 1 < high) {
Expand Down
2 changes: 1 addition & 1 deletion testdata/compiled_model/recipe.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"target":"predictor","sources":[{"name":"tu0","length":1307},{"name":"main","length":120}]}
{"target":"predictor","sources":[{"name":"tu0","length":4307},{"name":"main","length":130}]}
Loading

0 comments on commit 2e3f41e

Please sign in to comment.