haskell-te: d2b199662917dd49a11782a5252b55feec5ec8cb
1: { annotated, bash, buckets, buildEnv, cluster, concurrentQuickspec, fail,
2: format, glibcLocales, hashspecBench, jq, lib, mkBin, reduce-equations,
3: runCommand, runWeka, stdenv, testData, timeout, tipBenchmarks, withDeps, wrap,
4: writeScript }:
5: with builtins;
6: with lib;
7: rec {
8: benchVars = {
9: sampled = {
10: inherit (hashspecBench.benchVars.sampled) genInput;
11:
12: runner = wrap {
13: name = "mlspec-sampled-runner";
14: paths = [ ((import ../nix-support/augmentedHs.nix {
15: hsDir = "${tipBenchmarks.tip-benchmark-haskell}";
16: }).ghcWithPackages (h: map (n: h."${n}") [
17: "quickspec" "murmur-hash" "cereal" "mlspec-helper"
18: "tip-benchmark-sig" "runtime-arbitrary" "QuickCheck" "ifcxt"
19: "hashable" "mlspec"
20: ]))
21:
22: reduce-equations
23: buckets.hashes
24: runWeka ];
25: script = inEnvScript;
26: };
27: };
28: };
29:
30: ourEnv = writeScript "our-env.nix" ''
31: with import ${./..} {};
32: buildEnv {
33: name = "mlspecbench-env";
34: paths = [
35: ((import ${../nix-support/customHs.nix}).ghcWithPackages (h: [
36: h.tip-benchmark-sig h.mlspec
37: ]))
38: runWeka
39: jq
40: ];
41: }
42: '';
43:
44: inEnvScript = wrap {
45: name = "mlspecBench-inenvscript";
46: paths = [ bash concurrentQuickspec jq reduce-equations timeout ];
47: vars = {
48: NIX_EVAL_EXTRA_IMPORTS = ''[("tip-benchmark-sig", "A")]'';
49: SIMPLE = "1";
50: };
51: script = ''
52: #!${bash}/bin/bash
53: set -e
54: set -o pipefail
55:
56: # Perform clustering
57: CL=$(${cluster})
58:
59: clCount=$(echo "$CL" | jq 'map(.cluster) | max')
60:
61: export clCount
62:
63: if [[ -n "$EXPLORATION_MEM" ]]
64: then
65: echo "Limiting memory to '$EXPLORATION_MEM'" 1>&2
66: export MAX_KB="$EXPLORATION_MEM"
67: fi
68:
69: echo "$CL" | "${format.fromStdin}" |
70: withTimeout concurrentQuickspec | reduce-equations
71: '';
72: };
73:
74: mlGenInput = wrap {
75: name = "gen-input";
76: paths = [ bash jq tipBenchmarks.tools ];
77: vars = {
78: OUT_DIR = trace "FIXME: OUT_DIR/OUT_DIRS confusion" tipBenchmarks.tip-benchmark-haskell;
79: ANNOTATED = annotated {
80: pkgDir = toString tipBenchmarks.tip-benchmark-haskell;
81: };
82: filter = writeScript "filter.jq" ''
83: def mkId: {"name": .name, "package": .package, "module": .module};
84:
85: def keep($id): $keepers | map(. == $id) | any;
86:
87: def setQS: . + {"quickspecable": (.quickspecable and keep(mkId))};
88:
89: map(setQS)
90: '';
91: };
92: script = ''
93: #!${bash}/bin/bash
94: set -e
95: set -o pipefail
96:
97: # Sample some names, give the default module and package, then slurp
98: # into an array
99: echo "Running 'choose_sample $1 $2'" 1>&2
100: KEEPERS=$(choose_sample "$1" "$2" |
101: jq -R '{"name" : .,
102: "module" : "A",
103: "package" : "tip-benchmark-sig"}' |
104: jq -s '.')
105:
106: # Filters the signature to only those sampled in KEEPERS
107: jq --argjson keepers "$KEEPERS" -f "$filter" < "$ANNOTATED" |
108: jq 'map(select(.quickspecable))'
109: '';
110: };
111:
112: mlAllInput = wrap {
113: name = "all-input";
114: paths = [ fail ];
115: script = ''
116: #!${bash}/bin/bash
117: set -e
118:
119: [[ -n "$ANNOTATED" ]] || fail "Got no ANNOTATED"
120:
121: ANN_F=$(readlink -f ANNOTATED)
122: [[ -f "$ANN_F" ]] || fail "Annotated '$ANNOTATED' isn't a file (or link)"
123:
124: [[ -n "$OUT_DIR" ]] || fail "Got no OUT_DIR"
125:
126: OUT_D=$(readlink -f "$OUT_DIR")
127: [[ -d "$OUT_D" ]] || fail "OUT_DIR '$OUT_DIR' isn't a dir (or link)"
128:
129: cat "$ANNOTATED"
130: '';
131: };
132:
133: mls-untested = mkBin {
134: name = "mlspecBench";
135: vars = {
136: CMD = inEnvScript;
137: NIXENV = "import ${ourEnv}";
138: SKIP_NIX = "1";
139: LANG = "en_US.UTF-8";
140: LOCALE_ARCHIVE = "${glibcLocales}/lib/locale/locale-archive";
141: NIX_EVAL_HASKELL_PKGS = ../nix-support/customHs.nix;
142: NIX_PATH = concatStringsSep ":" [
143: "nixpkgs=${toString <nixpkgs>}"
144: "support=${toString ./..}"
145: ];
146: };
147: paths = [ bash haskellPkgToAsts hashspecBench.env ];
148: script = ''
149: #!${bash}/bin/bash
150: set -e
151:
152: ${hashspecBench.setUpDir}
153: export TEMPDIR="$DIR"
154: pushd "$DIR" > /dev/null
155: INPUT_TIP="$PWD/input_tip"
156: cat > "$INPUT_TIP"
157: OUT_DIR=$("$mkPkgInner")
158: export OUT_DIR
159: ANNOTATED=$(haskellPkgToAsts "$OUT_DIR")
160: export ANNOTATED
161: popd > /dev/null
162:
163: if [[ -n "$SAMPLE_SIZES" ]]
164: then
165: echo "Looping through sample sizes" 1>&2
166: for SAMPLE_SIZE in $SAMPLE_SIZES
167: do
168: echo "Limiting to a sample size of '$SAMPLE_SIZE'" 1>&2
169: export GEN_INPUT="${mlGenInput}"
170: INFO="$SAMPLE_SIZE" benchmark
171: done
172: else
173: echo "No sample size given, using whole signature" 1>&2
174: export GEN_INPUT="${mlAllInput}"
175: INFO="" benchmark
176: fi
177: '';
178: };
179:
180: MAX_SECS = "300";
181: testFile = name: path: runCommand "mls-${name}"
182: {
183: inherit MAX_SECS;
184: buildInputs = [ fail jq mls-untested ];
185: }
186: ''
187: echo "Running ${name} through mlspecBench" 1>&2
188: OUTPUT=$(mlspecBench < "${path}") ||
189: fail "Couldn't explore ${name}"
190:
191: T=$(echo "$OUTPUT" |
192: jq 'has("cmd") and has("info") and has("results")') ||
193: fail "Couldn't parse output\nSTART\n$OUTPUT\nEND"
194:
195: [[ "x$T" = "xtrue" ]] ||
196: fail "Required fields missing:\n$OUTPUT"
197: mkdir "$out"
198: '';
199:
200: mls = withDeps [
201: (testFile "list-full" ../benchmarks/list-full.smt2)
202: (testFile "nat-full" ../benchmarks/nat-full.smt2)
203: (testFile "nat-simple" ../benchmarks/nat-simple.smt2)
204: (attrValues (mapAttrs (name: runCommand name {
205: inherit MAX_SECS;
206: buildInputs = [ fail jq mls-untested tipBenchmarks.tools ];
207: }) {
208: canRun = ''
209: set -e
210: mlspecBench < "${testData.tip.test-theory}"
211: mkdir "$out"
212: '';
213:
214: outputIsJson = ''
215: set -e
216: OUTPUT=$(mlspecBench < ${testData.tip.test-theory}) ||
217: fail "Couldn't explore"
218: echo "$OUTPUT" | jq -e 'type | . == "object"') || {
219: echo "$OUTPUT" 1>&2
220: fail "Not an object"
221: }
222: '';
223:
224: haveEquations = ''
225: set -e
226: OUTPUT=$(mlspecBench < ${testData.tip.test-theory}) ||
227: fail "Couldn't explore"
228: echo "$OUTPUT" | jq -e 'has("results")' || {
229: echo "$OUTPUT" 1>&2
230: fail "Didn't find 'results'"
231: }
232: '';
233:
234: filterSamples =
235: with {
236: keepers = map (name: {
237: inherit name;
238: module = "A";
239: package = "tip-benchmark-sig";
240: }) [ "append" "constructorNil" "constructorCons" ];
241: };
242: ''
243: set -e
244:
245: BENCH_OUT=$(CLUSTERS=1 SAMPLE_SIZES="5" mlspecBench)
246:
247: # Get all the constant symbols in all equations
248: STDOUTS=$(echo "$BENCH_OUT" | jq -r '.results | .[] | .stdout') ||
249: fail "Couldn't get stdouts\n\n$BENCH_OUT"
250:
251: OUTPUTS=$(while read -r F
252: do
253: cat "$F"
254: done < <(echo "$STDOUTS")) ||
255: fail "Couldn't concat stdouts\n\n$BENCH_OUT\n\n$STDOUTS"
256:
257: EQS=$(echo "$OUTPUTS" | grep -v '^Depth') ||
258: fail "Couldn't get eqs\n\n$BENCH_OUT\n\n$OUTPUTS"
259:
260: NAMES=$(echo "$EQS" |
261: jq -r 'getpath(paths(type == "object" and .role == "constant")) |
262: .symbol' |
263: sort -u) || fail "Couldn't get names\n\n$BENCH_OUT\n\n$EQS"
264: SAMPLE=$(choose_sample 5 1)
265:
266: # Remove any names which appear in the sample
267: while read -r NAME
268: do
269: NAMES=$(echo "$NAMES" | grep -vFx "$NAME") || true
270: done < <(echo "$SAMPLE")
271:
272: # If there are any names remaining, they weren't in the sample
273: if echo "$NAMES" | grep '^.' > /dev/null
274: then
275: DBG="NAMES:\n$NAMES\n\nOUTPUT:\n$BENCH_OUT\nSAMPLE:\n$SAMPLE"
276: fail "Found names which aren't in sample\n$DBG"
277: fi
278:
279: mkdir "$out"
280: '';
281: }))
282: ] mls-untested;
283: }
Generated by git2html.