forked from NixOS/nixpkgs
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci/eval: use duckdb for creating diffs
- Loading branch information
Showing
7 changed files
with
221 additions
and
194 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| -- can create table from intermeidate outputs rather than needing jq to preprocess outpaths.json | ||
| CREATE MACRO raw(dir) AS TABLE | ||
| SELECT | ||
| cols[1] AS cmbAttr, | ||
| string_split(cols[-1], ';').list_sort() AS paths | ||
| FROM ( | ||
| SELECT c.regexp_split_to_array(' *') AS cols | ||
| FROM read_csv(dir || '/*/paths', header = FALSE, sep = '') AS t(c) | ||
| ); | ||
|
|
||
| CREATE TABLE before AS from raw('data/before'); | ||
| CREATE TABLE after AS from raw('data/after'); | ||
|
|
||
| /* | ||
| * expected file format: [ "aarch64-linux","aarch64-darwin", ... ] | ||
| */ | ||
| CREATE VIEW systems AS FROM 'data/systems.json' as t(target); | ||
|
|
||
| CREATE MACRO targetToOS(target) AS target.string_split('-')[-1]; | ||
| CREATE MACRO targetToMachine(target) AS target.string_split('-')[1]; | ||
| CREATE MACRO outDefault(paths) AS apply(paths, p -> if(p[1] = '/', 'out=' || p, p)); | ||
|
|
||
| /* | ||
| * Build the `changed` table: | ||
| * ┌─────────┬──────────────────────┬─────────┬─────────┬──────────────────────┐ | ||
| * │ action │ attr │ machine │ os │ paths │ | ||
| * ├─────────┼──────────────────────┼─────────┼─────────┼──────────────────────┤ | ||
| * │ change │ tambura │ x86_64 │ linux │ {out=/nix/store/rq… │ | ||
| * │ change │ tests.cc-wrapper.l… │ x86_64 │ darwin │ {out=/nix/store/aa… │ | ||
| * │ add │ cargo-llvm-cov │ aarch64 │ linux │ {out=/nix/store/ri… │ | ||
| * │ remove │ emscripten │ aarch64 │ darwin │ {out=/nix/store/mb… │ | ||
| * └───────────────────────────────────────────────────────────────────────────┘ | ||
| */ | ||
| CREATE TABLE changed AS | ||
| -- find paths that only exist in `after` table | ||
| WITH newPaths AS ( | ||
| SELECT | ||
| after.* | ||
| FROM after | ||
| LEFT JOIN before USING(paths) | ||
| WHERE before.paths IS NULL | ||
| ), | ||
| -- mark which paths were added or changed and append the removed attrs | ||
| combined AS ( | ||
| SELECT | ||
| newPaths.*, | ||
| if(before.cmbAttr IS NULL, 'add', 'change') AS action | ||
| FROM newPaths | ||
| LEFT JOIN before USING(cmbAttr) | ||
| UNION ALL | ||
| SELECT | ||
| before.*, | ||
| 'remove' AS action | ||
| FROM before | ||
| LEFT JOIN after USING(cmbAttr) | ||
| WHERE after.cmbAttr IS NULL | ||
| ) | ||
| -- split attr.target into columns `attr`, `machine`, `os` | ||
| -- prepare paths for json output: [ { "out" : "/nix/store/.. }, { ... } ] | ||
| -- filter out targets not in the system table | ||
| SELECT | ||
| action, | ||
| cmbAttr.regexp_replace('\.[^.]*$', '') AS attr, | ||
| target.targetToMachine() AS machine, | ||
| target.targetToOS() AS os, | ||
| map(paths.apply(c -> c[1]), paths.apply(c -> c[-1])) AS paths, | ||
| FROM ( | ||
| SELECT | ||
| * EXCLUDE(paths), | ||
| cmbAttr.string_split('.')[-1] AS target, | ||
| paths.outDefault().apply(c -> c.string_split('=')) AS paths | ||
| FROM combined | ||
| ) AS t | ||
| WHERE EXISTS (FROM systems WHERE systems.target = t.target); | ||
|
|
||
| /* | ||
| * Build the `rebuilds` table by counting added / changed paths. Attributes can | ||
| * have a many to one mapping to a path thus there are likely more changed | ||
| * attributes than paths. | ||
| * ┌────────┬─────────┬─────────┐ | ||
| * │ builds │ machine │ os │ | ||
| * ├────────┼─────────┼─────────┤ | ||
| * │ 64 │ aarch64 │ linux │ | ||
| * │ 44 │ x86_64 │ darwin │ | ||
| * │ 44 │ aarch64 │ darwin │ | ||
| * │ 64 │ x86_64 │ linux │ | ||
| * └────────┴─────────┴─────────┘ | ||
| */ | ||
| CREATE TABLE rebuilds AS | ||
| SELECT * REPLACE(max(builds) AS builds) -- filter out zero entries when builds > 0 | ||
| FROM ( | ||
| -- collect rebuilds for each system ignoring attributes mapping to same paths | ||
| SELECT | ||
| count(DISTINCT paths) AS builds, | ||
| machine, | ||
| os, | ||
| FROM changed | ||
| WHERE action != 'remove' | ||
| GROUP BY machine, os | ||
| UNION ALL | ||
| -- add zero builds for each system. Needed to prevent empty rows when the | ||
| -- rebuild count is zero. | ||
| SELECT | ||
| 0 AS builds, | ||
| target.targetToMachine() AS machine, | ||
| target.targetToOS() AS os, | ||
| FROM systems | ||
| ) | ||
| GROUP BY os, machine; | ||
|
|
||
| COPY (FROM rebuilds ORDER BY os, machine) TO 'rebuilds.json' (ARRAY); | ||
| COPY (FROM changed ORDER BY attr, machine, os) TO 'changed.json' (ARRAY); |
Oops, something went wrong.