From 5c311011c61ea6cbc1b7469bb7e9774fdeb67a56 Mon Sep 17 00:00:00 2001 From: bld Date: Fri, 22 Jan 2021 18:21:55 +0000 Subject: [PATCH 1/2] Validation charts --- src/witan/send/vis/validation_charts.clj | 22 ++++ src/witan/send/vis/validation_charts/ncy.clj | 120 ++++++++++++++++++ .../send/vis/validation_charts/needs.clj | 76 +++++++++++ .../send/vis/validation_charts/settings.clj | 76 +++++++++++ 4 files changed, 294 insertions(+) create mode 100644 src/witan/send/vis/validation_charts.clj create mode 100644 src/witan/send/vis/validation_charts/ncy.clj create mode 100644 src/witan/send/vis/validation_charts/needs.clj create mode 100644 src/witan/send/vis/validation_charts/settings.clj diff --git a/src/witan/send/vis/validation_charts.clj b/src/witan/send/vis/validation_charts.clj new file mode 100644 index 0000000..7c9524e --- /dev/null +++ b/src/witan/send/vis/validation_charts.clj @@ -0,0 +1,22 @@ +(ns witan.send.vis.validation-charts + (:require [witan.send.vis.validation-charts.ncy :as ncy] + [witan.send.vis.validation-charts.needs :as needs] + [witan.send.vis.validation-charts.settings :as settings])) + +(defn charts [transitions {:keys [ncy-titles-and-sets + need-titles-and-sets + setting-titles-and-sets + output-dir] + :or {ncy-titles-and-sets ncy/titles-and-sets}}] + (let [ncy-charts (ncy/ncy-charts transitions ncy-titles-and-sets) + needs-charts (needs/needs-charts transitions need-titles-and-sets) + settings-charts (settings/settings-charts transitions setting-titles-and-sets)] + + (ncy/->excel output-dir ncy-charts) + (needs/->excel output-dir needs-charts) + (settings/->excel output-dir settings-charts) + + {:transitions transitions + :ncy-charts ncy-charts + :needs-charts needs-charts + :settings-charts settings-charts})) diff --git a/src/witan/send/vis/validation_charts/ncy.clj b/src/witan/send/vis/validation_charts/ncy.clj new file mode 100644 index 0000000..c565268 --- /dev/null +++ b/src/witan/send/vis/validation_charts/ncy.clj @@ -0,0 +1,120 @@ +(ns witan.send.vis.validation-charts.ncy + (:require [clojure.set :as set] + [witan.send.chart :as wsc] + [witan.send.domain.academic-years :as ay] + [witan.send.driver.transitions :as dt] + [witan.send.vis.census.academic-years :as vis.cay] + [witan.send.vis.ingest.transitions :as vit])) + +(def all-ages (into [] (concat ay/early-years ay/key-stage-1 ay/key-stage-2 ay/key-stage-3 ay/key-stage-4 ay/key-stage-5 #_ay/ncy-15+))) + +(def titles-and-sets + [["Early Years" ay/early-years] + ["Key Stage 1" ay/key-stage-1] + ["Key Stage 2" ay/key-stage-2] + ["Key Stage 3" ay/key-stage-3] + ["Key Stage 4" ay/key-stage-4] + ["Key Stage 5" ay/key-stage-5] + ["NCY 15+" ay/ncy-15+] + ["All NCYs" all-ages]]) + +(defn overall-chart [{:keys [colors-and-points titles-and-sets census]}] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets titles-and-sets} + census + true)) + +(defn ncy-by-setting [{:keys [colors-and-points census settings]}] + (mapcat + (fn [setting] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets [[(str "NCYs for Setting " setting) + all-ages]]} + (into [] + (filter #(= setting (:setting %))) + census) + true)) + settings)) + +(defn ncy-by-need [{:keys [colors-and-points needs census]}] + (mapcat + (fn [need] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets [[(str "NCYs for Need " need) + all-ages]]} + (into [] + (filter #(= need (:need %))) + census) + true)) + needs)) + +(defn stayers [{:keys [colors-and-points transitions titles-and-sets]}] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Stayers in " title) pred]) titles-and-sets)} + (->> transitions + (filter dt/stayer?) + (vit/->census)) + true)) + +(defn movers-in [{:keys [colors-and-points transitions titles-and-sets]}] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Movers in " title) pred]) titles-and-sets)} + (->> transitions + (filter dt/mover?) + (vit/->census)) + true)) + +(defn joiners-to [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Joiners to " title) pred]) titles-and-sets)} + (into [] + (comp + (filter dt/joiner?) + (map #(select-keys % [:calendar-year :setting-2 :need-2 :academic-year-2])) + (map #(set/rename-keys % {:setting-2 :setting + :need-2 :need + :academic-year-2 :academic-year}))) + transitions) + true)) + +(defn leavers-from [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cay/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Leavers from " title) pred]) titles-and-sets)} + (into [] + (comp + (filter dt/leaver?) + (map #(select-keys % [:calendar-year :setting-1 :need-1 :academic-year-1])) + (map #(set/rename-keys % {:setting-1 :setting + :need-1 :need + :academic-year-1 :academic-year}))) + transitions) + true)) + +(defn ncy-charts + ([transitions override-titles-and-sets] + (let [census (vit/->census transitions) + config {:colors-and-points (wsc/domain-colors-and-points :academic-year census) + :titles-and-sets override-titles-and-sets + :needs (into (sorted-set) (map :need) census) + :settings (into (sorted-set) (map :setting) census) + :transitions transitions + :census census}] + (into [] + (mapcat (fn [f] (f config))) + [overall-chart + ncy-by-setting + ncy-by-need + stayers + movers-in + joiners-to + leavers-from]))) + ([transitions] + (ncy-charts transitions titles-and-sets))) + +(defn ->pngs [output-dir charts] + (run! (partial wsc/save-chart-by-title (str output-dir "charts/ncy-")) charts) + charts) + +(defn ->excel [output-dir charts] + (wsc/save-workbook (str output-dir "charts/census-ncy.xlsx") (wsc/->workbook charts)) + charts) diff --git a/src/witan/send/vis/validation_charts/needs.clj b/src/witan/send/vis/validation_charts/needs.clj new file mode 100644 index 0000000..60577bb --- /dev/null +++ b/src/witan/send/vis/validation_charts/needs.clj @@ -0,0 +1,76 @@ +(ns witan.send.vis.validation-charts.needs + (:require [clojure.set :as set] + [witan.send.chart :as wsc] + [witan.send.driver.transitions :as dt] + [witan.send.vis.census.needs :as vis.cn] + [witan.send.vis.ingest.transitions :as vit])) + +(defn all-needs [{:keys [colors-and-points titles-and-sets census]}] + (vis.cn/charts {:colors-and-points colors-and-points + :titles-and-sets titles-and-sets} + census + true)) + +(defn stayers-in [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cn/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Stayers in " title) pred]) titles-and-sets)} + (->> transitions + (filter dt/stayer?) + (vit/->census)) + true)) + +(defn movers-in [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cn/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Movers in " title) pred]) titles-and-sets)} + (->> transitions + (filter dt/mover?) + (vit/->census)) + true)) + +(defn joiners-to [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cn/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Joiners to " title) pred]) titles-and-sets)} + (into [] + (comp + (filter dt/joiner?) + (map #(select-keys % [:calendar-year :setting-2 :need-2 :academic-year-2])) + (map #(set/rename-keys % {:setting-2 :setting + :need-2 :need + :academic-year-2 :academic-year}))) + transitions) + true)) + +(defn leavers-from [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cn/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Leavers from " title) pred]) titles-and-sets)} + (into [] + (comp + (filter dt/leaver?) + (map #(select-keys % [:calendar-year :setting-1 :need-1 :academic-year-1])) + (map #(set/rename-keys % {:setting-1 :setting + :need-1 :need + :academic-year-1 :academic-year}))) + transitions) + true)) + +(defn needs-charts [transitions titles-and-sets] + (let [census (vit/->census transitions) + config {:colors-and-points (wsc/domain-colors-and-points :need census) + :titles-and-sets titles-and-sets + :transitions transitions + :census census}] + (into [] + (mapcat (fn [f] (f config))) + [all-needs + stayers-in + movers-in + joiners-to + leavers-from]))) + +(defn ->pngs [output-dir charts] + (run! (partial wsc/save-chart-by-title (str output-dir "charts/needs-")) charts) + charts) + +(defn ->excel [output-dir charts] + (wsc/save-workbook (str output-dir "charts/census-needs.xlsx") (wsc/->workbook charts)) + charts) diff --git a/src/witan/send/vis/validation_charts/settings.clj b/src/witan/send/vis/validation_charts/settings.clj new file mode 100644 index 0000000..598cd3a --- /dev/null +++ b/src/witan/send/vis/validation_charts/settings.clj @@ -0,0 +1,76 @@ +(ns witan.send.vis.validation-charts.settings + (:require [clojure.set :as set] + [witan.send.chart :as wsc] + [witan.send.driver.transitions :as dt] + [witan.send.vis.census.settings :as vis.cs] + [witan.send.vis.ingest.transitions :as vit])) + +(defn all-settings [{:keys [colors-and-points titles-and-sets census]}] + (vis.cs/charts {:colors-and-points colors-and-points + :titles-and-sets titles-and-sets} + census + true)) + +(defn stayers [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cs/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Stayers in " title) pred]) titles-and-sets)} + (->> transitions + (filter dt/stayer?) + (vit/->census)) + true)) + +(defn movers-in [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cs/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Movers in " title) pred]) titles-and-sets)} + (->> transitions + (filter dt/mover?) + (vit/->census)) + true)) + +(defn joiners [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cs/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Joiners to " title) pred]) titles-and-sets)} + (into [] + (comp + (filter dt/joiner?) + (map #(select-keys % [:calendar-year :setting-2 :need-2 :academic-year-2])) + (map #(set/rename-keys % {:setting-2 :setting + :need-2 :need + :academic-year-2 :academic-year}))) + transitions) + true)) + + +(defn leavers [{:keys [colors-and-points titles-and-sets transitions]}] + (vis.cs/charts {:colors-and-points colors-and-points + :titles-and-sets (mapv (fn [[title pred]] [(str "Leavers from " title) pred]) titles-and-sets)} + (into [] + (comp + (filter dt/leaver?) + (map #(select-keys % [:calendar-year :setting-1 :need-1 :academic-year-1])) + (map #(set/rename-keys % {:setting-1 :setting + :need-1 :need + :academic-year-1 :academic-year}))) + transitions) + true)) + +(defn settings-charts [transitions titles-and-sets] + (let [census (vit/->census transitions) + config {:colors-and-points (wsc/domain-colors-and-points :setting census) + :titles-and-sets titles-and-sets + :transitions transitions + :census census}] + (into [] + (mapcat (fn [f] (f config))) + [all-settings + stayers + joiners + leavers]))) + +(defn ->pngs [output-dir charts] + (run! (partial wsc/save-chart-by-title (str output-dir "charts/settings-")) charts) + charts) + +(defn ->excel [output-dir charts] + (wsc/save-workbook (str output-dir "charts/census-settings.xlsx") (wsc/->workbook charts)) + charts) From 36e47c5dd6095674dc7662bcd16e6cd4e288cf80 Mon Sep 17 00:00:00 2001 From: bld Date: Fri, 22 Jan 2021 18:22:03 +0000 Subject: [PATCH 2/2] Projection charts --- src/witan/send/vis/projection_charts.clj | 97 ++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 src/witan/send/vis/projection_charts.clj diff --git a/src/witan/send/vis/projection_charts.clj b/src/witan/send/vis/projection_charts.clj new file mode 100644 index 0000000..c076b1f --- /dev/null +++ b/src/witan/send/vis/projection_charts.clj @@ -0,0 +1,97 @@ +(ns witan.send.vis.projection-charts + (:require [clojure.core.async :as a] + [witan.send.chart :as wsc] + [witan.send.main :as wsm] + [witan.send.vis.general-population :as vis.gp] + [witan.send.vis.ingest.transitions :as vis.transitions] + [witan.send.vis.output-ay :as vis.oay] + [witan.send.vis.output-cost :as vis.ocost] + [witan.send.vis.output-count :as vis.ocount] + [witan.send.vis.output-need :as vis.on] + [witan.send.vis.output-setting :as vis.os] + [witan.send.vis.output-setting-cost :as vis.osc])) + +(defn charts [config-file {:keys [setting-titles-and-sets need-titles-and-sets]}] + (let [config (wsm/read-config config-file) + output-dir (str (:project-dir config) "/" (get-in config [:output-parameters :output-dir])) + historical-transitions (vis.transitions/historical (str (:project-dir config) "/" (get-in config [:file-inputs :transitions]))) + gen-pop-data (into [] + (comp + ;; only a subset so the charts line up + (filter #(<= 2015 (:calendar-year %))) + (filter #(< (:academic-year %) 21))) + (vis.gp/general-population (str (:project-dir config) "/" (get-in config [:file-inputs :population])))) + census (vis.transitions/->census historical-transitions) + settings-lookup (into {} + (map (fn [x] [(:setting x) (:setting x)])) + census) + needs-lookup (into {} + (map (fn [x] [(:setting x) (:setting x)])) + census) + output-ay (future + (vis.oay/charts + (vis.transitions/ay-counts-per-calendar-year historical-transitions) + (into [] + (filter #(< (:academic-year %) 21)) + (vis.oay/output-ay (str output-dir "/" vis.oay/output-ay-file))))) + output-count (future + (vis.ocount/chart + "2020 Baseline" + (vis.transitions/total-population-per-calendar-year historical-transitions) + (vis.ocount/output-count (str output-dir "/" vis.ocount/output-count-file)))) + output-cost (future + (vis.ocost/chart + "2020 Baseline" + (vis.ocost/output-cost (str output-dir "/" vis.ocost/output-cost-file)))) + output-need (future + (vis.on/charts + needs-lookup + (vis.transitions/needs-counts-per-calendar-year historical-transitions) + (vis.on/output-need (str output-dir "/" vis.on/output-need-file)) + need-titles-and-sets)) + output-setting (future + (vis.os/charts + settings-lookup + (vis.transitions/settings-counts-per-calendar-year historical-transitions) + (vis.os/output-setting (str output-dir "/" vis.os/output-setting-file)) + setting-titles-and-sets)) + output-setting-cost (vis.osc/charts + (vis.osc/output-setting-cost (str output-dir "/" vis.osc/output-setting-cost-file)) + setting-titles-and-sets) + general-population (future (vis.gp/charts gen-pop-data))] + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/ay-")) @output-ay) + (wsc/save-workbook (str output-dir "/charts/output-ay.xlsx") (wsc/->workbook @output-ay))) + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/count-")) @output-count) + (wsc/save-workbook (str output-dir "/charts/output-count.xlsx") (wsc/->workbook @output-count))) + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/cost-")) @output-cost) + (wsc/save-workbook (str output-dir "/charts/output-cost.xlsx") (wsc/->workbook @output-cost))) + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/need-")) @output-need) + (wsc/save-workbook (str output-dir "/charts/output-need.xlsx") (wsc/->workbook @output-need))) + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/setting-")) @output-setting) + (wsc/save-workbook (str output-dir "/charts/output-setting.xlsx") (wsc/->workbook @output-setting))) + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/setting-cost-")) output-setting-cost) + (wsc/save-workbook (str output-dir "/charts/output-setting-cost.xlsx") (wsc/->workbook output-setting-cost))) + + (a/thread + ;; (run! (partial wsc/save-chart-by-title (str output-dir "/charts/general-population-")) @general-population) + (wsc/save-workbook (str output-dir "/charts/general-population.xlsx") (wsc/->workbook @general-population))) + + {:output-ay @output-ay + :output-count @output-count + :output-cost @output-cost + :output-need @output-need + :output-setting @output-setting + :output-setting-cost output-setting-cost + :general-population @general-population}))