From 8d6acb2502e848cf2fb04a7207a50f854582c3a8 Mon Sep 17 00:00:00 2001 From: Tobias Mirco Schiffmann Date: Thu, 25 Jan 2024 13:56:20 +0100 Subject: [PATCH] Configs for all projects - working for picosat and gzip --- varats/varats/projects/c_projects/gzip.py | 107 ++++++++++++++++++- varats/varats/projects/c_projects/picosat.py | 102 ++++++------------ varats/varats/projects/c_projects/x264.py | 37 ++++++- varats/varats/projects/c_projects/xz.py | 1 + 4 files changed, 176 insertions(+), 71 deletions(-) diff --git a/varats/varats/projects/c_projects/gzip.py b/varats/varats/projects/c_projects/gzip.py index f3b2a106d..0ddc4638a 100644 --- a/varats/varats/projects/c_projects/gzip.py +++ b/varats/varats/projects/c_projects/gzip.py @@ -4,7 +4,7 @@ import benchbuild as bb from benchbuild.command import Command, SourceRoot, WorkloadSet -from benchbuild.source import HTTPMultiple +from benchbuild.source import HTTPMultiple, HTTPUntar from benchbuild.utils.cmd import make, mkdir from benchbuild.utils.revision_ranges import block_revisions, RevisionRange from benchbuild.utils.settings import get_number_of_jobs @@ -78,6 +78,13 @@ class Gzip(VProject, ReleaseProviderHook): "download/v0.6.0" }, files=["countries-land-1km.geo.json", "countries-land-1m.geo.json"] + ), + HTTPUntar( + local="cantrbry.tar.gz", + remote={ + "1.0": + "http://corpus.canterbury.ac.nz/resources/cantrbry.tar.gz" + } ) ] @@ -86,7 +93,105 @@ class Gzip(VProject, ReleaseProviderHook): 'gettext', 'texinfo', 'rsync' ) + files = ["alice29.txt", "asyoulik.txt", "cp.html", "fields.c", "grammar.lsp", + "kennedy.xls", "lcet10.txt", "plrabn12.txt", "ptt5", "sum", "xargs.1"] + + configs = [ + ["-1"], + ["-5"], + ["-9"], + ["--recursive", "-1"], + ["--recursive", "-5"], + ["--recursive", "-9"], + ["--verbose", "-1"], + ["--verbose", "-5"], + ["--verbose", "-9"], + ["--quiet", "-1"], + ["--quiet", "-5"], + ["--quiet", "-9"], + ["-S", "-1"], + ["-S", "-5"], + ["-S", "-9"], + ["--test", "-1"], + ["--test", "-5"], + ["--test", "-9"], + ["--recursive", "--verbose", "-1"], + ["--recursive", "--verbose", "-5"], + ["--recursive", "--verbose", "-9"], + ["--recursive", "--quiet", "-1"], + ["--recursive", "--quiet", "-5"], + ["--recursive", "--quiet", "-9"], + ["--recursive", "-S", "-1"], + ["--recursive", "-S", "-5"], + ["--recursive", "-S", "-9"], + ["--recursive", "--test", "-1"], + ["--recursive", "--test", "-5"], + ["--recursive", "--test", "-9"], + ["--verbose", "--quiet", "-1"], + ["--verbose", "--quiet", "-5"], + ["--verbose", "--quiet", "-9"], + ["--verbose", "-S", "-1"], + ["--verbose", "-S", "-5"], + ["--verbose", "-S", "-9"], + ["--verbose", "--test", "-1"], + ["--verbose", "--test", "-5"], + ["--verbose", "--test", "-9"], + ["--quiet", "-S", "-1"], + ["--quiet", "-S", "-5"], + ["--quiet", "-S", "-9"], + ["--quiet", "--test", "-1"], + ["--quiet", "--test", "-5"], + ["--quiet", "--test", "-9"], + ["-S", "--test", "-1"], + ["-S", "--test", "-5"], + ["-S", "--test", "-9"] + ] + + commands = [] + + + for file in files: + for i, config in enumerate(configs): + command = None + length = len(config) + if length == 1: + command = Command( + SourceRoot("gzip") / RSBinary("gzip"), + config[0], + "--force", # needed because BB creates symlinks for the inputs + "--keep", # needed for repeating with the same workload + "cantrbry.tar.gz/" + file, + label=file + "-" + str(i) + ' '.join(config), + creates=["cantrbry.tar.gz/" + file + ".gz"] + ) + elif length == 2: + command = Command( + SourceRoot("gzip") / RSBinary("gzip"), + config[0], + config[1], + "--force", # needed because BB creates symlinks for the inputs + "--keep", # needed for repeating with the same workload + "cantrbry.tar.gz/" + file, + label=file + "-" + str(i) + ' '.join(config), + creates=["cantrbry.tar.gz/" + file + ".gz"] + ) + else: + command = Command( + SourceRoot("gzip") / RSBinary("gzip"), + config[0], + config[1], + config[2], + "--force", # needed because BB creates symlinks for the inputs + "--keep", # needed for repeating with the same workload + "cantrbry.tar.gz/" + file, + label=file + "-" + str(i) + ' '.join(config), + creates=["cantrbry.tar.gz/" + file + ".gz"] + ) + + commands.append(command) + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): commands, WorkloadSet(WorkloadCategory.SMALL): [ Command( SourceRoot("gzip") / RSBinary("gzip"), diff --git a/varats/varats/projects/c_projects/picosat.py b/varats/varats/projects/c_projects/picosat.py index 71de307e6..c565ef497 100644 --- a/varats/varats/projects/c_projects/picosat.py +++ b/varats/varats/projects/c_projects/picosat.py @@ -52,91 +52,59 @@ class PicoSAT(VProject, ReleaseProviderHook): shallow=False ), FeatureSource(), - HTTP( - local="example.cnf", - remote={ - "1.0": - "https://github.com/se-sic/picoSAT-mirror/releases/" - "download/picoSAT-965/example.cnf" - } - ), HTTPUntar( - local="abw-N-bcsstk07.mtx-w44.cnf", + local="uf50-218", remote={ "1.0": - "https://github.com/se-sic/picoSAT-mirror/releases/" - "download/picoSAT-965/abw-N-bcsstk07.mtx-w44.cnf.tar.gz" + "https://www.cs.ubc.ca/~hoos/SATLIB/Benchmarks/SAT/RND3SAT/uf50-218.tar.gz" } ), HTTPUntar( - local="traffic_kkb_unknown.cnf", + local="uf250-1065", remote={ - "1.0": - "https://github.com/se-sic/picoSAT-mirror/releases/" - "download/picoSAT-965/traffic_kkb_unknown.cnf.tar.gz" + "2.0": + "https://www.cs.ubc.ca/~hoos/SATLIB/Benchmarks/SAT/RND3SAT/uf250-1065.tar.gz" } ), HTTPUntar( - local="UNSAT_H_instances_childsnack_p11.hddl_1.cnf", + local="uf150-645", remote={ - "1.0": - "https://github.com/se-sic/picoSAT-mirror/releases/" - "download/picoSAT-965/" - "UNSAT_H_instances_childsnack_p11.hddl_1.cnf.tar.gz" - } - ), - HTTPUntar( - local="UNSAT_H_instances_childsnack_p12.hddl_1.cnf", - remote={ - "1.0": - "https://github.com/se-sic/picoSAT-mirror/releases/" - "download/picoSAT-965/" - "UNSAT_H_instances_childsnack_p12.hddl_1.cnf.tar.gz" + "3.0": + "https://www.cs.ubc.ca/~hoos/SATLIB/Benchmarks/SAT/RND3SAT/uf150-645.tar.gz" } ), ] - WORKLOADS = { - WorkloadSet(WorkloadCategory.EXAMPLE): [ - Command( - SourceRoot("picosat") / RSBinary("picosat"), - "example.cnf", - label="example.cnf", - ) - ], - WorkloadSet(WorkloadCategory.SMALL): [ - Command( - SourceRoot("picosat") / RSBinary("picosat"), - "aim-100-1_6-no-1.cnf", - label="aim-100-1-6-no-1.cnf", - ) - ], - WorkloadSet(WorkloadCategory.MEDIUM): [ - Command( - SourceRoot("picosat") / RSBinary("picosat"), - "traffic_kkb_unknown.cnf/traffic_kkb_unknown.cnf", - label="traffic-kkb-unknow.cnf", - ), - Command( + commands = [] + + for i in range(1, 200): + name = "uf50-0" + str(i) + commands.append(Command( SourceRoot("picosat") / RSBinary("picosat"), - "abw-N-bcsstk07.mtx-w44.cnf/abw-N-bcsstk07.mtx-w44.cnf", - label="abw-N-bcsstk07.mtx-w44.cnf", - ), - ], - WorkloadSet(WorkloadCategory.LARGE): [ - Command( + "uf50-218/" + name + ".cnf", + label=name, + )) + + for i in range(1, 100): + name = "uf250-0" + str(i) + commands.append(Command( SourceRoot("picosat") / RSBinary("picosat"), - "UNSAT_H_instances_childsnack_p11.hddl_1.cnf/" - "UNSAT_H_instances_childsnack_p11.hddl_1.cnf", - label="UNSAT-H-instances-childsnack-p11.hddl-1.cnf", - ), - Command( + "uf250-1065/ai/hoos/Shortcuts/UF250.1065.100/" + name + ".cnf", + label=name, + )) + + for i in range(1, 100): + name = "uf150-0" + str(i) + commands.append(Command( SourceRoot("picosat") / RSBinary("picosat"), - "UNSAT_H_instances_childsnack_p12.hddl_1.cnf/" - "UNSAT_H_instances_childsnack_p12.hddl_1.cnf", - label="UNSAT-H-instances-childsnack-p12.hddl-1.cnf", - ) - ], + "uf150-1065/ai/hoos/Shortcuts/UF150.645.100/" + name + ".cnf", + label=name, + )) + + + + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): commands } @staticmethod diff --git a/varats/varats/projects/c_projects/x264.py b/varats/varats/projects/c_projects/x264.py index e0b1dcee0..330d9226d 100644 --- a/varats/varats/projects/c_projects/x264.py +++ b/varats/varats/projects/c_projects/x264.py @@ -4,6 +4,9 @@ import benchbuild as bb from benchbuild.utils.cmd import make from benchbuild.utils.revision_ranges import block_revisions, GoodBadSubgraph +from benchbuild.source import HTTP, HTTPUntar +from benchbuild.command import Command, SourceRoot, WorkloadSet +from varats.experiment.workload_util import RSBinary, WorkloadCategory from benchbuild.utils.settings import get_number_of_jobs from plumbum import local @@ -46,9 +49,35 @@ class X264(VProject): limit=None, shallow=False ) + ), + HTTP( + local="akiyo_cif.y4m", + remote={ + "1.0": + "https://media.xiph.org/video/derf/y4m/akiyo_cif.y4m" + } + ), + HTTPUntar( + local="subset1-y4m.tar.gz", + remote={ + "1.0": + "https://media.xiph.org/video/derf/subset1-y4m.tar.gz" + } ) ] + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + Command( + SourceRoot("x264") / RSBinary("x264"), + "-o End_of_Show_3.mkv", + "subset1-y4m.tar.gz/subset1-y4m/End_of_Show_3.y4m", + label="End_of_Show_3", + creates=["End_of_Show_3.mkv"] + ), + ] + } + CONTAINER = get_base_image(ImageBase.DEBIAN_10) @staticmethod @@ -69,6 +98,8 @@ def compile(self) -> None: x264_version_source = local.path(self.source_of_primary) x264_version = ShortCommitHash(self.version_of_primary) + print(x264_version_source) + fpic_revisions = get_all_revisions_between( "5dc0aae2f900064d1f58579929a2285ab289a436", "290de9638e5364c37316010ac648a6c959f6dd26", ShortCommitHash, @@ -83,13 +114,13 @@ def compile(self) -> None: if x264_version in fpic_revisions: self.cflags += ["-fPIC"] - clang = bb.compiler.cc(self) + compiler = bb.compiler.cc(self) with local.cwd(x264_version_source): - with local.env(CC=str(clang)): + with local.env(CC=str(compiler)): configure_flags = ["--disable-asm"] if x264_version in ldflags_revisions: configure_flags.append("--extra-ldflags=\"-static\"") - bb.watch(local["./configure"])(configure_flags) + bb.watch(local["./configure"])(["--disable-asm"]) bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) verify_binaries(self) diff --git a/varats/varats/projects/c_projects/xz.py b/varats/varats/projects/c_projects/xz.py index 3d1a580ed..3855618f5 100644 --- a/varats/varats/projects/c_projects/xz.py +++ b/varats/varats/projects/c_projects/xz.py @@ -88,6 +88,7 @@ class Xz(VProject): VCommand( SourceRoot("xz") / RSBinary("xz"), "-k", + "-z", # Use output_param to ensure input file # gets appended after all arguments. output_param=["{output}"],