haskell-te: a5d6fc7da012996f563b5c2b439bd34ed8f3a3df
1: { annotated, bash, buckets, buildEnv, concurrentQuickspec, fail,
2: glibcLocales, jq, mlspecBench, nix, nixEnv, reduce-equations, runCommand,
3: stdenv, testData, timeout, tipBenchmarks, wrap, writeScript }:
4:
5: with builtins;
6: rec {
7: benchVars = {
8: sampled = {
9: runner = wrap {
10: name = "hashspec-sampled-runner";
11: paths = [ ((import ../nix-support/augmentedHs.nix {
12: hsDir = "${tipBenchmarks.tip-benchmark-haskell}";
13: }).ghcWithPackages (h: map (n: h."${n}") [
14: "quickspec" "murmur-hash" "cereal" "mlspec-helper"
15: "tip-benchmark-sig" "runtime-arbitrary" "QuickCheck" "ifcxt"
16: "hashable" "mlspec"
17: ]))
18:
19: reduce-equations
20: buckets.hashes
21: fail
22: concurrentQuickspec
23: ];
24: vars = {
25: NIX_EVAL_EXTRA_IMPORTS = ''[("tip-benchmark-sig", "A")]'';
26: };
27: script = ''
28: #!${bash}/bin/bash
29: set -e
30: [[ -n "$TEMPDIR" ]] || fail "No TEMPDIR given"
31:
32: [[ -n "$MAX_KB" ]] || {
33: echo "Setting default memory limit of 2GB" 1>&2
34: export MAX_KB=2000000
35: }
36:
37: hashBucket | concurrentQuickspec | reduce-equations
38: '';
39: };
40:
41: genInput = wrap {
42: name = "hashspec-sampled-gen-input";
43: paths = [ fail jq tipBenchmarks.tools ];
44: vars = {
45: OUT_DIRS = toJSON [tipBenchmarks.tip-benchmark-haskell];
46:
47: ANNOTATED = annotated {
48: pkgDir = toString tipBenchmarks.tip-benchmark-haskell;
49: };
50:
51: FILTER = writeScript "filter.jq" ''
52: def mkId: {"name": .name, "package": .package, "module": .module};
53:
54: def keep($id): $keepers | map(. == $id) | any;
55:
56: def setQS: . + {"quickspecable": (.quickspecable and keep(mkId))};
57:
58: map(setQS)
59: '';
60: };
61: script = ''
62: #!${bash}/bin/bash
63: set -e
64: set -o pipefail
65:
66: [[ -n "$ANNOTATED" ]] || fail "No ANNOTATED given"
67: [[ -n "$OUT_DIRS" ]] || fail "No OUT_DIRS given"
68:
69: # Give sampled names a module and package, then slurp into an array
70: KEEPERS=$(jq -R '{"name" : .,
71: "module" : "A",
72: "package" : "tip-benchmark-sig"}' |
73: jq -s '.')
74:
75: # Filters the signature to only those sampled in KEEPERS
76: jq --argjson keepers "$KEEPERS" -f "$FILTER" < "$ANNOTATED" |
77: jq 'map(select(.quickspecable))'
78: '';
79: };
80: };
81: };
82:
83: setUpDir = ''
84: [[ -n "$DIR" ]] || {
85: echo "No DIR given to work in, using current directory $PWD" 1>&2
86: DIR="$PWD"
87: }
88: export DIR
89: '';
90:
91: mkPkgInner = wrap {
92: name = "mkPkgInner";
93: paths = [ fail ];
94: vars = {
95: MAKE_PACKAGE = wrap {
96: name = "make-haskell-package";
97: paths = [ tipBenchmarks.tools ];
98: script = ''
99: OUT_DIRS="[\"$PWD\"]" full_haskell_package < "$INPUT_TIP"
100: '';
101: };
102: };
103: script = ''
104: #!${bash}/bin/bash
105: set -e
106: [[ -n "$INPUT_TIP" ]] || fail "No INPUT_TIP given, aborting"
107:
108: echo "Creating Haskell package" 1>&2
109: mkdir "generated-haskell-package"
110: pushd "generated-haskell-package" > /dev/null
111: "$MAKE_PACKAGE" || fail "Failed to create Haskell package"
112: popd > /dev/null
113: echo "Created Haskell package" 1>&2
114: '';
115: };
116:
117: env = buildEnv {
118: name = "te-env";
119: paths = [ jq nix tipBenchmarks.tools ];
120: };
121:
122: hs-untested = mkBin {
123: name = "hashspecBench";
124: paths = [ bash env haskellPkgToAsts jq ];
125: vars = {
126: CMD = wrap {
127: name = "hashspecBench-inenvscript";
128: paths = [
129: bash concurrentQuickspec reduce-equations timeout buckets.hashes
130: ];
131: vars = {
132: NIX_EVAL_EXTRA_IMPORTS = ''[("tip-benchmark-sig", "A")]'';
133: };
134: script = ''
135: #!${bash}/bin/bash
136:
137: if [[ -n "$EXPLORATION_MEM" ]]
138: then
139: echo "Limiting memory to '$EXPLORATION_MEM'" 1>&2
140: export MAX_KB="$EXPLORATION_MEM"
141: fi
142:
143: echo "Exploring" 1>&2
144: hashBucket | withTimeout concurrentQuickspec | reduce-equations
145: '';
146: };
147: NIXENV = "import ${mlspecBench.ourEnv}";
148: SKIP_NIX = "1";
149: LANG = "en_US.UTF-8";
150: LOCALE_ARCHIVE = "${glibcLocales}/lib/locale/locale-archive";
151: NIX_EVAL_HASKELL_PKGS = ../nix-support/customHs.nix;
152: NIX_PATH = concatStringsSep ":" [
153: "nixpkgs=${toString <nixpkgs>}"
154: "support=${toString ./..}"
155: ];
156: };
157: script = ''
158: #!${bash}/bin/bash
159: set -e
160:
161: ${setUpDir}
162: export TEMPDIR="$DIR"
163: pushd "$DIR" > /dev/null
164: INPUT_TIP="$PWD/input_tip"
165: cat > "$INPUT_TIP"
166: OUT_DIR=$("$mkPkgInner")
167: export OUT_DIR
168: ANNOTATED=$(haskellPkgToAsts "$OUT_DIR")
169: export ANNOTATED
170: popd > /dev/null
171:
172: # FIXME: OUT_DIR/OUT_DIRS confusion here
173:
174: if [[ -n "$SAMPLE_SIZES" ]]
175: then
176: echo "Looping through sample sizes" 1>&2
177: for SAMPLE_SIZE in $SAMPLE_SIZES
178: do
179: echo "Limiting to a sample size of '$SAMPLE_SIZE'" 1>&2
180: export GEN_INPUT="${mlspecBench.mlGenInput}"
181: INFO="$SAMPLE_SIZE" benchmark
182: done
183: else
184: echo "No sample size given, using whole signature" 1>&2
185: export GEN_INPUT="${mlspecBench.mlAllInput}"
186: INFO="" benchmark
187: fi
188: '';
189: };
190:
191: MAX_SECS = "300";
192: testFile = name: path: runCommand "hs-${name}"
193: {
194: inherit MAX_SECS;
195: buildInputs = [ fail jq hs-untested ];
196: }
197: ''
198: set -e
199: echo "Running ${name} through hashspecBench" 1>&2
200: OUTPUT=$(hashspecBench < "${path}") ||
201: fail "Couldn't explore ${name}"
202:
203: T=$(echo "$OUTPUT" |
204: jq 'has("cmd") and has("info") and has("results")') ||
205: fail "Couldn't parse output\nSTART\n$OUTPUT\nEND"
206:
207: [[ "x$T" = "xtrue" ]] ||
208: fail "Required fields missing:\n$OUTPUT"
209:
210: mkdir "$out"
211: '';
212:
213: hs = withDeps
214: [
215: (testFile "list-full" ../benchmarks/list-full.smt2)
216: (testFile "nat-full" ../benchmarks/nat-full.smt2)
217: (testFile "nat-simple" ../benchmarks/nat-simple.smt2)
218: (attrValues (mapAttrs
219: (name: runCommand name {
220: inherit MAX_SECS;
221: buildInputs = [ fail hs-untested tipBenchmarks.tools ];
222: })
223: {
224: canRun = ''
225: set -e
226: hashspecBench < "${testData.tip.test-theory}"
227: mkdir "$out"
228: '';
229:
230: outputIsJson = ''
231: set -e
232: OUTPUT=$(hashspecBench < ${testData.tip.test-theory})
233:
234: TYPE=$(echo "$OUTPUT" | jq -r 'type') ||
235: fail "START OUTPUT\n$OUTPUT\nEND OUTPUT"
236:
237: [[ "x$TYPE" = "xobject" ]] ||
238: fail "START OUTPUT\n$OUTPUT\nEND OUTPUT\nGot '$TYPE' not object"
239:
240: mkdir "$out"
241: '';
242:
243: haveEquations = ''
244: set -e
245: OUTPUT=$(hashspecBench < ${testData.tip.test-theory}) || exit 1
246: CHECK=$(echo "$OUTPUT" | jq 'has("results")') || exit 1
247: [[ "x$CHECK" = "xtrue" ]] ||
248: fail "Didn't find 'results' in\n$OUTPUT"
249: mkdir "$out"
250: '';
251:
252: filterSamples =
253: with {
254: keepers = map (name: {
255: inherit name;
256: module = "A";
257: package = "tip-benchmark-sig";
258: }) [ "append" "constructorNil" "constructorCons" ];
259: };
260: ''
261: set -e
262:
263: BENCH_OUT=$(CLUSTERS=1 SAMPLE_SIZES="5" hashspecBench)
264:
265: # Get all the constant symbols in all equations
266: STDOUTS=$(echo "$BENCH_OUT" | jq -r '.results | .[] | .stdout') ||
267: fail "Couldn't get stdouts\n\n$BENCH_OUT"
268:
269: OUTPUTS=$(while read -r F
270: do
271: cat "$F"
272: done < <(echo "$STDOUTS")) ||
273: fail "Couldn't concat stdouts\n\n$BENCH_OUT\n\n$STDOUTS"
274:
275: EQS=$(echo "$OUTPUTS" | grep -v '^Depth') ||
276: fail "Couldn't get eqs\n\n$BENCH_OUT\n\n$OUTPUTS"
277:
278: NAMES=$(echo "$EQS" |
279: jq -r 'getpath(paths(type == "object" and .role == "constant"))
280: | .symbol' | sort -u) ||
281: fail "Couldn't get names\n\n$BENCH_OUT\n\n$EQS"
282: SAMPLE=$(choose_sample 5 1)
283:
284: # Remove any names which appear in the sample
285: while read -r NAME
286: do
287: NAMES=$(echo "$NAMES" | grep -vFx "$NAME") || true
288: done < <(echo "$SAMPLE")
289:
290: # If there are any names remaining, they weren't in the sample
291: if echo "$NAMES" | grep '^.' > /dev/null
292: then
293: DBG="NAMES:\n$NAMES\n\nOUTPUT:\n$BENCH_OUT\nSAMPLE:\n$SAMPLE"
294: fail "Found names which aren't in sample\n$DBG"
295: fi
296:
297: mkdir "$out"
298: '';
299: }))
300: ]
301: hs-untested;
302: }
Generated by git2html.