diff --git a/Project.toml b/Project.toml index a05fc1b..3936a4f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,19 +1,14 @@ name = "CEEDesigns" uuid = "e939450b-799e-4198-a5f5-3f2f7fb1c671" -version = "0.3.5" +version = "0.3.6" [deps] -Clustering = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" -HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -LibPQ = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MCTS = "e12ccd36-dcad-5f33-8774-9175229e7b33" MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" -POMDPSimulators = "e0d0a172-29c6-5d4e-96d0-f262df5d01fd" POMDPTools = "7588e00f-9cae-40de-98dc-e0c70c48cdd7" POMDPs = "a93abf59-7444-517b-a68a-c42f96afdd7d" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" @@ -25,24 +20,19 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" [compat] -julia = "1.9" -Plots = "1.39" -ScientificTypes = "3.0" -POMDPTools = "0.1" -DataFrames = "1.6" -HTTP = "1.10" -LinearAlgebra = "1.9" -LibPQ = "1.17" Combinatorics = "1.0" -Statistics = "1.9" -Random = "1.9" -Reexport = "1.2" -Distances = "0.10" -POMDPs = "0.9" +DataFrames = "1.6" JSON = "0.21" -Clustering = "0.15" +LinearAlgebra = "1.9" MCTS = "0.5" MLJ = "0.20" +POMDPTools = "0.1" +POMDPs = "0.9" +Plots = "1.39" +Random = "1.9" +Reexport = "1.2" Requires = "1.3" -POMDPSimulators = "0.3" +ScientificTypes = "3.0" +Statistics = "1.9" StatsBase = "0.34" +julia = "1.9" diff --git a/docs/Project.toml b/docs/Project.toml index 408fb3d..c3784be 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -2,14 +2,21 @@ BetaML = "024491cd-cc6b-443e-8034-08ea7eb7db2b" CEEDesigns = "e939450b-799e-4198-a5f5-3f2f7fb1c671" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" +Copulas = "ae264745-0b69-425e-9d9d-cf662c5eec93" D3Trees = "e3df1716-f71e-5df9-9e2d-98e193103c45" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433" Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" MCTS = "e12ccd36-dcad-5f33-8774-9175229e7b33" MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" MLJModels = "d491faf4-2d78-11e9-2867-c94bc002c0b7" +POMDPTools = "7588e00f-9cae-40de-98dc-e0c70c48cdd7" +POMDPs = "a93abf59-7444-517b-a68a-c42f96afdd7d" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd" diff --git a/docs/make.jl b/docs/make.jl index 7cc3e8b..bd6bfea 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -3,8 +3,13 @@ using CEEDesigns # Literate for tutorials const literate_dir = joinpath(@__DIR__, "..", "tutorials") -const tutorials_src = - ["StaticDesigns.jl", "StaticDesignsFiltration.jl", "GenerativeDesigns.jl"] +const tutorials_src = [ + "SimpleStatic.jl", + "SimpleGenerative.jl", + "StaticDesigns.jl", + "StaticDesignsFiltration.jl", + "GenerativeDesigns.jl", +] const generated_dir = joinpath(@__DIR__, "src", "tutorials/") # copy tutorials src @@ -29,6 +34,8 @@ end pages = [ "index.md", "Tutorials" => [ + "tutorials/SimpleStatic.md", + "tutorials/SimpleGenerative.md", "tutorials/StaticDesigns.md", "tutorials/StaticDesignsFiltration.md", "tutorials/GenerativeDesigns.md", diff --git a/docs/src/api.md b/docs/src/api.md index 069b2b3..05469e2 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -33,7 +33,7 @@ CEEDesigns.GenerativeDesigns.efficient_value CEEDesigns.GenerativeDesigns.DistanceBased CEEDesigns.GenerativeDesigns.QuadraticDistance CEEDesigns.GenerativeDesigns.DiscreteDistance -CEEDesigns.GenerativeDesigns.MahalanobisDistance +CEEDesigns.GenerativeDesigns.SquaredMahalanobisDistance CEEDesigns.GenerativeDesigns.Exponential ``` diff --git a/docs/src/index.md b/docs/src/index.md index 43603fb..21f0c10 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -7,11 +7,11 @@ A decision-making framework for the cost-efficient design of experiments, balanc ``` ## Static experimental designs -Here we assume that the same experimental design will be used for a population of examined entities, hence the word 'static'. +Here we assume that the same experimental design will be used for a population of examined entities, hence the word "static". For each subset of experiments, we consider an estimate of the value of acquired information. To give an example, if a set of experiments is used to predict the value of a specific target variable, our framework can leverage a built-in integration with [MLJ.jl](https://github.com/alan-turing-institute/MLJ.jl) to estimate predictive accuracies of machine learning models fitted over subset of experimental features. -In the cost-sensitive setting of CEEDesigns, a user provides the monetary cost and execution time of each experiment. Given the constraint on the maximum number of parallel experiments along with a fixed tradeoff between monetary cost and execution time, we devise an arrangement of each subset of experiments such that the expected combined cost is minimized. +In the cost-sensitive setting of CEEDesigns.jl`, a user provides the monetary cost and execution time of each experiment. Given the constraint on the maximum number of parallel experiments along with a fixed tradeoff between monetary cost and execution time, we devise an arrangement of each subset of experiments such that the expected combined cost is minimized. Assuming the information values and optimized experimental costs for each subset of experiments, we then generate a set of cost-efficient experimental designs. @@ -23,7 +23,7 @@ Assuming the information values and optimized experimental costs for each subset We consider 'personalized' experimental designs that dynamically adjust based on the evidence gathered from the experiments. This approach is motivated by the fact that the value of information collected from an experiment generally differs across subpopulations of the entities involved in the triage process. -At the beginning of the triage process, an entity's prior data is used to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model likely experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts (sampled from a generative model, hence the word 'generative'), we adjust the continuation based on this evidence. +At the beginning of the triage process, an entity's prior data is used to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model likely experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts (sampled from a generative model, hence the word "generative"), we adjust the continuation based on this evidence. ```@raw html code diff --git a/docs/src/tutorials/GenerativeDesigns.jl b/docs/src/tutorials/GenerativeDesigns.jl index d74f31f..2d1f4f1 100644 --- a/docs/src/tutorials/GenerativeDesigns.jl +++ b/docs/src/tutorials/GenerativeDesigns.jl @@ -1,90 +1,11 @@ # # Heart Disease Triage Meets Generative Modeling # Consider again a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer; instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. +# For details on the theoretical background and notation, please see our tutorial on [generative experimental designs](SimpleGenerative.md). This tutorial +# is a concrete application of the tools described in that document. # Importantly, we aim to design personalized adaptive policies for each patient. At the beginning of the triage process, we use a patient's prior data, such as sex, age, or type of chest pain, to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model probable experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts, we adjust the continuation based on this evidence. -# The personalized experimental designs are motivated by the fact that the value of information collected from an experiment often differs across subsets of the entities involved in the triage process. - -# ![information value matrix](assets/information_value_matrix.png) - -# In the context of static designs, where each individual from the 'Population' undergoes the same triage process, 'Experiment C' would contribute the maximum information value. On the other hand, if we have the capability to adapt the triage process specifically for 'Population 1' and 'Population 2', we would choose 'Experiment 1' and 'Experiment 2' respectively, thereby improving the value of information acquired. - -# ## Theoretical Framework - -# Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$, and let $y$ denote the target variable that we want to predict. - -# We conceptualize the triage as a Markov decision process, in which we iteratively choose to conduct a subset of experiments $S \subseteq E$ and then, based on the experimental evidence, update our belief about the distribution of outcomes for the experiments that have not yet been conducted. - -# Within the framework, -# - _state_ is modeled as the set of experiments conducted so far along with the acquired experimental evidence and accumulated costs; -# - _actions_ are subsets of experiments that have not yet been conducted; the size of these subsets is restricted by the maximum number of parallel experiments. - -# Importantly, the outcome of a set $S$ of experiments is modeled as a random variable $e_S$, conditioned on the current state, i.e., combined evidence. This means that if in a given state outcomes from experiments in $S \subseteq E$ are available, the outcome of experiments in $S' \subseteq E \setminus S$ is drawn from a posterior $r \sim q(e_{S'} | e_S)$. - -# We do not claim to know the 'best' way to define the posterior $q$. Instead, our approach is generic and allows us to consider any generative function that takes the current state and the set of experiments to be conducted, and returns a sample drawn from the implicit, theoretical posterior of the selected experiments. - -# The information value associated with the state, derived from experimental evidence, can be modeled through any statistical or information-theoretic measure such as the variance or uncertainty associated with the target variable posterior $q(y|e_S)$. - -# For example, consider [EDDI: Efficient Dynamic Discovery of High-Value Information with Partial VAE](https://arxiv.org/pdf/1809.11142.pdf), where the authors use a VAE approach to model the posterior distribution and utilize Kullback-Leibler divergence to model the information gain. - -# #### Distance-Based Sampling from Historical Data - -# We implemented an approach that iteratively distributes a belief over historical entities likely to be similar to the entity currently being tested. As a result, we sample from the outputs of these historical entities, weighted by the probabilistic weight assigned by our system of belief. In doing so, we produce a specific model for $q(e_{S^{'}} | e_S)$ and $q(y | e_S)$. - -# More specifically, consider a dataset of historical outputs across $m$ features $X = \{x_1, \ldots, x_m\}$ for $l$ entities (with entities and features representing rows and columns, respectively). Let's denote the target variable we want to predict as $y$. - -# We assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. - -# In what follows, we discuss the assignment of probabilistic weights $w_j$ to each entity (or row in the dataset). - -# In the initial state, when there is no experimental evidence gathered yet, we assign the weights according to a predetermined prior. - -# If this is not the case, we have to compute the weights using particular distance and similarity functionals. - -# For each feature $x\in X$, we consider a function $\rho_x$, which measures the distance between two outputs. By default, we consider: -# - Rescaled Kronecker delta (i.e., $\rho(x, y)=0$ only when $x=y$, and $\rho(x, y)= \lambda$ under any other circumstances, with $\lambda > 0$) for discrete features (i.e., features whose types are modeled as `MultiClass` type in [ScientificTypes.jl](https://github.com/JuliaAI/ScientificTypes.jl)); -# - Rescaled squared distance $\rho(x, y) = λ \frac{(x - y)^2}{2\sigma_2}$, where $\sigma_2$ is the variance of the feature column, estimated with respect to the prior for continuous features. - -# Therefore, given an experimental state with readouts over the feature set $F \subseteq X$, we can calculate the total distance from the entity recorded in the $j$-th row as $d_j = \sum_{x\in F} \rho_x (\hat x, x_j)$, where $\hat x$ and $x_j$ denote the readout for feature $x$ for the entity being tested and the entity recorded in $j$-th column, respectively. - -# Alternatively, we could use the [Mahalanobis distance](https://en.wikipedia.org/wiki/Mahalanobis_distance#Definition). - -# Next, we convert distances $d_j$ into probabilistic weights $w_j$. By default, we use a rescaled exponential function, i.e., we put $w_j = \exp(-\lambda d_j)$ for some $\lambda>0$. Notably, $\lambda$'s value determines how belief is distributed across the historical entities. Larger values of $\lambda$ concentrate the belief tightly around the 'closest' historical entities, while smaller values distribute more belief to more distant entities. - -# Importantly, the proper choice of the distance functionals and the 'similarity' functional discussed above is a question of hyper-optimization. - -# Assuming the weights $w_j$ have been assigned, we can sample an index $\hat j \in \{ 1, \ldots, l\}$ according to these weights. To draw a sample $\hat x$ from feature $x\in X$, we let $\hat x$ be equal to the value in the column associated with feature $x$ for the $\hat j$-th row (entity). We can also take a sample from the target variable in the same way. - -# ### Objective Sense -# The reward and stopping condition of the triage process can be interpreted in various ways. -# - The triage may continue until the uncertainty about the posterior distribution of the target variable falls below a certain level. Our aim is to minimize the anticipated combined monetary cost and execution time of the triage (considered as a 'negative' reward). If all experiments are conducted without reaching below the required uncertainty level, or if the maximum number of experiments is exceeded, we penalize this scenario with a 'minus infinite' reward. -# - We may aim to minimize the expected uncertainty while being constrained by the costs of the experiment. -# - Alternatively, we could maximize the value of experimental evidence, adjusted for the incurred experimental costs. - -# ### Policy Search -# Standard MDP algorithms can be used to solve this problem (offline learning) or construct the policy (online learning) for the sequential decision-making. - -# Our MDP's state space is finite-dimensional but generally continuous due to the allowance of continuous features, which complicates the problem and few algorithms specialize in this area. - -# We used the Double Progressive Widening Algorithm for this task as detailed in [A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation](https://arxiv.org/abs/1405.5498). - -# In a nutshell, the Double Progressive Widening (DPW) algorithm is designed for online learning in complex environments, particularly those known as Continuous Finite-dimensional Markov Decision Processes where the state space is continuous. The key idea behind DPW is to progressively expand the search tree during the Monte Carlo Tree Search (MCTS) process. The algorithm does so by dynamically and selectively adding states and actions to the tree based on defined heuristics. - -# In the context of online learning, this algorithm addresses the complexity and challenges of real-time decision-making in domains with a large or infinite number of potential actions. As information is gathered in actual runtime, the algorithm explores and exploits this information to make optimal or near-optimal decisions. In other words, DPW permits the learning process to adapt on-the-fly as more data is made available, making it an effective tool for the dynamic and uncertain nature of online environments. - -# In particular, at the core of the Double Progressive Widening (DPW) algorithm are several key components, including expansion, search, and rollout. - -# The search component is where the algorithm sifts through the search tree to determine the most promising actions or states to explore next. By using exploration-exploitation strategies, it can effectively balance its efforts between investigating previously successful actions and venturing into unexplored territories. - -# The expansion phase is where the algorithm grows the search tree by adding new nodes, representing new state-action pairs, to the tree. This is done based on a predefined rule that dictates when and how much the tree should be expanded. This allows the algorithm to methodically discover and consider new potential actions without becoming overwhelmed with choices. - -# Lastly, the rollout stage, also known as a simulation stage, is where the algorithm plays out a series of actions to the end of a game or scenario using a specific policy (like a random policy). The results of these rollouts are then used to update the estimates of the values of state-action pairs, increasing the accuracy of future decisions. - -# ![One iteration of the MCTS algorithm, taken from https://ieeexplore.ieee.org/document/6145622](assets/mcts.png) - -# In the above figure, nodes represent states of the decision process, while edges correspond to actions connecting these states. - # ## Heart Disease Dataset # In this tutorial, we consider a dataset that includes 11 clinical features along with a binary variable indicating the presence of heart disease in patients. The dataset can be found at [Kaggle: Heart Failure Prediction](https://www.kaggle.com/datasets/fedesoriano/heart-failure-prediction). It utilizes heart disease datasets from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/dataset/45/heart+disease). @@ -120,7 +41,7 @@ using CEEDesigns, CEEDesigns.GenerativeDesigns # As previously discussed, we provide a dataset of historical records, the target variable, along with an information-theoretic measure to quantify the uncertainty about the target variable. # In what follows, we obtain three functions: -# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator, # - `uncertainty`: this is a function of `evidence`, # - `weights`: this represents a function of `evidence` that distributes probabilistic weights across the rows in the dataset. @@ -131,7 +52,7 @@ using CEEDesigns, CEEDesigns.GenerativeDesigns (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); @@ -145,7 +66,7 @@ categorical_feats = setdiff(names(data), numeric_feats) DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), distance = merge( Dict(c => DiscreteDistance() for c in categorical_feats), @@ -153,14 +74,14 @@ DistanceBased( ), ); -# You can also use the Mahalanobis distance (`MahalanobisDistance(; diagonal)`). For example, we could write: +# You can also use the squared Mahalanobis distance (`SquaredMahalanobisDistance(; diagonal)`). As the squared Mahalanobis distance only works with numeric features, we have to select a few, along with the target variable. For example, we could write: DistanceBased( - data[!, ["RestingBP", "MaxHR", "Cholesterol", "FastingBS", "HeartDisease"]]; # the Mahalanobis distance only works with numeric features, so we selected a few, along with the target variable + data[!, ["RestingBP", "MaxHR", "Cholesterol", "FastingBS", "HeartDisease"]]; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), - distance = MahalanobisDistance(; diagonal = 1), + distance = SquaredMahalanobisDistance(; diagonal = 1), ); # The package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. @@ -268,7 +189,7 @@ experiments = Dict( "HeartDisease" => (100.0, 100.0), ) -# We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. +# We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time. In our case, we aim to minimize the execution time. ## minimize time, two concurrent experiments at maximum seed!(1) @@ -297,7 +218,7 @@ plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") # In this experimental setup, we aim to maximize the value of experimental evidence, adjusted for the incurred experimental costs. -# For this purpose, we need to specify a function that quantifies the 'value' of decision-process making state, modeled as a tuple of experimental evidence and costs. +# For this purpose, we need to specify a function that quantifies the "value" of decision-process making state, modeled as a tuple of experimental evidence and costs. value = function (evidence, (monetary_cost, execution_time)) return (1 - uncertainty(evidence)) - (0.005 * sum(monetary_cost)) diff --git a/docs/src/tutorials/GenerativeDesigns.md b/docs/src/tutorials/GenerativeDesigns.md index e56edc7..db36e29 100644 --- a/docs/src/tutorials/GenerativeDesigns.md +++ b/docs/src/tutorials/GenerativeDesigns.md @@ -5,90 +5,11 @@ EditURL = "GenerativeDesigns.jl" # Heart Disease Triage Meets Generative Modeling Consider again a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer; instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. +For details on the theoretical background and notation, please see our tutorial on [generative experimental designs](SimpleGenerative.md). This tutorial +is a concrete application of the tools described in that document. Importantly, we aim to design personalized adaptive policies for each patient. At the beginning of the triage process, we use a patient's prior data, such as sex, age, or type of chest pain, to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model probable experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts, we adjust the continuation based on this evidence. -The personalized experimental designs are motivated by the fact that the value of information collected from an experiment often differs across subsets of the entities involved in the triage process. - -![information value matrix](assets/information_value_matrix.png) - -In the context of static designs, where each individual from the 'Population' undergoes the same triage process, 'Experiment C' would contribute the maximum information value. On the other hand, if we have the capability to adapt the triage process specifically for 'Population 1' and 'Population 2', we would choose 'Experiment 1' and 'Experiment 2' respectively, thereby improving the value of information acquired. - -## Theoretical Framework - -Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$, and let $y$ denote the target variable that we want to predict. - -We conceptualize the triage as a Markov decision process, in which we iteratively choose to conduct a subset of experiments $S \subseteq E$ and then, based on the experimental evidence, update our belief about the distribution of outcomes for the experiments that have not yet been conducted. - -Within the framework, -- _state_ is modeled as the set of experiments conducted so far along with the acquired experimental evidence and accumulated costs; -- _actions_ are subsets of experiments that have not yet been conducted; the size of these subsets is restricted by the maximum number of parallel experiments. - -Importantly, the outcome of a set $S$ of experiments is modeled as a random variable $e_S$, conditioned on the current state, i.e., combined evidence. This means that if in a given state outcomes from experiments in $S \subseteq E$ are available, the outcome of experiments in $S' \subseteq E \setminus S$ is drawn from a posterior $r \sim q(e_{S'} | e_S)$. - -We do not claim to know the 'best' way to define the posterior $q$. Instead, our approach is generic and allows us to consider any generative function that takes the current state and the set of experiments to be conducted, and returns a sample drawn from the implicit, theoretical posterior of the selected experiments. - -The information value associated with the state, derived from experimental evidence, can be modeled through any statistical or information-theoretic measure such as the variance or uncertainty associated with the target variable posterior $q(y|e_S)$. - -For example, consider [EDDI: Efficient Dynamic Discovery of High-Value Information with Partial VAE](https://arxiv.org/pdf/1809.11142.pdf), where the authors use a VAE approach to model the posterior distribution and utilize Kullback-Leibler divergence to model the information gain. - -#### Distance-Based Sampling from Historical Data - -We implemented an approach that iteratively distributes a belief over historical entities likely to be similar to the entity currently being tested. As a result, we sample from the outputs of these historical entities, weighted by the probabilistic weight assigned by our system of belief. In doing so, we produce a specific model for $q(e_{S^{'}} | e_S)$ and $q(y | e_S)$. - -More specifically, consider a dataset of historical outputs across $m$ features $X = \{x_1, \ldots, x_m\}$ for $l$ entities (with entities and features representing rows and columns, respectively). Let's denote the target variable we want to predict as $y$. - -We assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. - -In what follows, we discuss the assignment of probabilistic weights $w_j$ to each entity (or row in the dataset). - -In the initial state, when there is no experimental evidence gathered yet, we assign the weights according to a predetermined prior. - -If this is not the case, we have to compute the weights using particular distance and similarity functionals. - -For each feature $x\in X$, we consider a function $\rho_x$, which measures the distance between two outputs. By default, we consider: -- Rescaled Kronecker delta (i.e., $\rho(x, y)=0$ only when $x=y$, and $\rho(x, y)= \lambda$ under any other circumstances, with $\lambda > 0$) for discrete features (i.e., features whose types are modeled as `MultiClass` type in [ScientificTypes.jl](https://github.com/JuliaAI/ScientificTypes.jl)); -- Rescaled squared distance $\rho(x, y) = λ \frac{(x - y)^2}{2\sigma_2}$, where $\sigma_2$ is the variance of the feature column, estimated with respect to the prior for continuous features. - -Therefore, given an experimental state with readouts over the feature set $F \subseteq X$, we can calculate the total distance from the entity recorded in the $j$-th row as $d_j = \sum_{x\in F} \rho_x (\hat x, x_j)$, where $\hat x$ and $x_j$ denote the readout for feature $x$ for the entity being tested and the entity recorded in $j$-th column, respectively. - -Alternatively, we could use the [Mahalanobis distance](https://en.wikipedia.org/wiki/Mahalanobis_distance#Definition). - -Next, we convert distances $d_j$ into probabilistic weights $w_j$. By default, we use a rescaled exponential function, i.e., we put $w_j = \exp(-\lambda d_j)$ for some $\lambda>0$. Notably, $\lambda$'s value determines how belief is distributed across the historical entities. Larger values of $\lambda$ concentrate the belief tightly around the 'closest' historical entities, while smaller values distribute more belief to more distant entities. - -Importantly, the proper choice of the distance functionals and the 'similarity' functional discussed above is a question of hyper-optimization. - -Assuming the weights $w_j$ have been assigned, we can sample an index $\hat j \in \{ 1, \ldots, l\}$ according to these weights. To draw a sample $\hat x$ from feature $x\in X$, we let $\hat x$ be equal to the value in the column associated with feature $x$ for the $\hat j$-th row (entity). We can also take a sample from the target variable in the same way. - -### Objective Sense -The reward and stopping condition of the triage process can be interpreted in various ways. -- The triage may continue until the uncertainty about the posterior distribution of the target variable falls below a certain level. Our aim is to minimize the anticipated combined monetary cost and execution time of the triage (considered as a 'negative' reward). If all experiments are conducted without reaching below the required uncertainty level, or if the maximum number of experiments is exceeded, we penalize this scenario with a 'minus infinite' reward. -- We may aim to minimize the expected uncertainty while being constrained by the costs of the experiment. -- Alternatively, we could maximize the value of experimental evidence, adjusted for the incurred experimental costs. - -### Policy Search -Standard MDP algorithms can be used to solve this problem (offline learning) or construct the policy (online learning) for the sequential decision-making. - -Our MDP's state space is finite-dimensional but generally continuous due to the allowance of continuous features, which complicates the problem and few algorithms specialize in this area. - -We used the Double Progressive Widening Algorithm for this task as detailed in [A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation](https://arxiv.org/abs/1405.5498). - -In a nutshell, the Double Progressive Widening (DPW) algorithm is designed for online learning in complex environments, particularly those known as Continuous Finite-dimensional Markov Decision Processes where the state space is continuous. The key idea behind DPW is to progressively expand the search tree during the Monte Carlo Tree Search (MCTS) process. The algorithm does so by dynamically and selectively adding states and actions to the tree based on defined heuristics. - -In the context of online learning, this algorithm addresses the complexity and challenges of real-time decision-making in domains with a large or infinite number of potential actions. As information is gathered in actual runtime, the algorithm explores and exploits this information to make optimal or near-optimal decisions. In other words, DPW permits the learning process to adapt on-the-fly as more data is made available, making it an effective tool for the dynamic and uncertain nature of online environments. - -In particular, at the core of the Double Progressive Widening (DPW) algorithm are several key components, including expansion, search, and rollout. - -The search component is where the algorithm sifts through the search tree to determine the most promising actions or states to explore next. By using exploration-exploitation strategies, it can effectively balance its efforts between investigating previously successful actions and venturing into unexplored territories. - -The expansion phase is where the algorithm grows the search tree by adding new nodes, representing new state-action pairs, to the tree. This is done based on a predefined rule that dictates when and how much the tree should be expanded. This allows the algorithm to methodically discover and consider new potential actions without becoming overwhelmed with choices. - -Lastly, the rollout stage, also known as a simulation stage, is where the algorithm plays out a series of actions to the end of a game or scenario using a specific policy (like a random policy). The results of these rollouts are then used to update the estimates of the values of state-action pairs, increasing the accuracy of future decisions. - -![One iteration of the MCTS algorithm, taken from https://ieeexplore.ieee.org/document/6145622](assets/mcts.png) - -In the above figure, nodes represent states of the decision process, while edges correspond to actions connecting these states. - ## Heart Disease Dataset In this tutorial, we consider a dataset that includes 11 clinical features along with a binary variable indicating the presence of heart disease in patients. The dataset can be found at [Kaggle: Heart Failure Prediction](https://www.kaggle.com/datasets/fedesoriano/heart-failure-prediction). It utilizes heart disease datasets from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/dataset/45/heart+disease). @@ -131,7 +52,7 @@ using CEEDesigns, CEEDesigns.GenerativeDesigns As previously discussed, we provide a dataset of historical records, the target variable, along with an information-theoretic measure to quantify the uncertainty about the target variable. In what follows, we obtain three functions: -- `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +- `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator, - `uncertainty`: this is a function of `evidence`, - `weights`: this represents a function of `evidence` that distributes probabilistic weights across the rows in the dataset. @@ -143,7 +64,7 @@ You can specify the method for computing the distance using the `distance` keywo (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); nothing #hide @@ -160,7 +81,7 @@ categorical_feats = setdiff(names(data), numeric_feats) DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), distance = merge( Dict(c => DiscreteDistance() for c in categorical_feats), @@ -170,15 +91,15 @@ DistanceBased( nothing #hide ```` -You can also use the Mahalanobis distance (`MahalanobisDistance(; diagonal)`). For example, we could write: +You can also use the squared Mahalanobis distance (`SquaredMahalanobisDistance(; diagonal)`). As the squared Mahalanobis distance only works with numeric features, we have to select a few, along with the target variable. For example, we could write: ````@example GenerativeDesigns DistanceBased( - data[!, ["RestingBP", "MaxHR", "Cholesterol", "FastingBS", "HeartDisease"]]; # the Mahalanobis distance only works with numeric features, so we selected a few, along with the target variable + data[!, ["RestingBP", "MaxHR", "Cholesterol", "FastingBS", "HeartDisease"]]; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), - distance = MahalanobisDistance(; diagonal = 1), + distance = SquaredMahalanobisDistance(; diagonal = 1), ); nothing #hide ```` @@ -311,7 +232,7 @@ experiments = Dict( ) ```` -We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. +We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time. In our case, we aim to minimize the execution time. ````@example GenerativeDesigns # minimize time, two concurrent experiments at maximum @@ -345,7 +266,7 @@ plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") In this experimental setup, we aim to maximize the value of experimental evidence, adjusted for the incurred experimental costs. -For this purpose, we need to specify a function that quantifies the 'value' of decision-process making state, modeled as a tuple of experimental evidence and costs. +For this purpose, we need to specify a function that quantifies the "value" of decision-process making state, modeled as a tuple of experimental evidence and costs. ````@example GenerativeDesigns value = function (evidence, (monetary_cost, execution_time)) diff --git a/docs/src/tutorials/SimpleGenerative.jl b/docs/src/tutorials/SimpleGenerative.jl new file mode 100644 index 0000000..aaf8624 --- /dev/null +++ b/docs/src/tutorials/SimpleGenerative.jl @@ -0,0 +1,512 @@ +# # Generative Experimental Designs + +# This document describes the theoretical background behind tools in CEEDesigns.jl for generative experimental designs +# and demonstrates uses on synthetic data examples. + +# ## Setting + +# Generative experimental designs differ from static designs in that the experimental design is specific for (or "personalized") to an incoming entity. +# Personalized cost-efficient experimental designs for the new entity are made based on existing information (if any) about the new entity, +# and from posterior distributions of unobserved features of that entity conditional on its observed features and historical data. +# The entities may be patients, molecular compounds, or any other objects where one has a store of historical data +# and seeks to find efficient experimental designs to learn more about a new arrival. +# +# The personalized experimental designs are motivated by the fact that the value of information collected from an experiment +# often differs across subsets of the population of entities. + +# ![information value matrix](assets/information_value_matrix.png) + +# In the context of static designs, we do not aspire to capture variation in information gain across different entities. Instead, we assume all entities come from a "Population" with a uniform information gain, in which case "Experiment C" would provide the maximum information value. +# On the other hand, if we have the ability to discern if the entity belong to subpopulations "Population 1" or "Population 2," then we can tailor our +# design to suggest either "Experiment A" or "Experiment B." Clearly, in the limit of a maximally heterogenous population, each +# entity has its own "row." Our tools are able to handle the entire spectrum of such scenarios though distance based similarity, +# described below. + +# ## Theoretical Framework + +# ### Historical Data + +# Like static designs, we consider a set $E$ of $n$ experiments, each with an associated tuple $(m_{e},t_{e})$ of monetary and +# time costs (for more details on experiments and arrangements, see the tutorial on [static experimental designs](SimpleStatic.md)). +# +# Additionally, consider a historical dataset giving measurements on $m$ features $X = \{x_1, \ldots, x_m\}$ for $l$ entities +# (with entities and features representing rows and columns, respectively). Each experiment $e$ may yield measurements on some +# subset of features $(X_{e}\subseteq X)$. +# +# Furthermore there is an additional column $y$ which is a target variable we want to predict (CEEDesigns.jl may allow $y$ +# to be a vector, but we assume it is scalar here for ease of presentation). +# +# Letting $m=3$, then the historical dataset may be visualized as the following table: + +# ![historical data](assets/generative_historical.png) + +# ### New Entity + +# When a new entity arrives, it will have an unknown outcome $y$ and unknown values of some or all features $x_{i} \in X$. +# We call the _state_ of the new entity the set of experiments conducted upon it so far (if any), along with the +# measurements on any features produced by those experiments (if any), called _evidence_. +# +# Consider the case where there are $n=3$ experiments, each of which yields a measurement on a single feature. Then +# the state of a new entity arriving for which $e_{1}$ has already been performed will look like: + +# ![state](assets/generative_state.png) + +# Letting $S$ denote the set of experiments which have been performed so far, then $S^{\prime}=E\S$ are unperformed experiments. +# Then, _actions_ one can perform to learn more about the new entity are subsets of $S^{\prime}$. The size of subsets +# is limited by the maximum number of parallel experiments. +# +# In our running example, if the maximum number of parallel experiments is at least 2, then the available actions are: + +# ![actions](assets/generative_actions.png) + +# ### Posterior Distributions + +# Let $e_{S}$ be a random variable denoting the outcome of some experiments $S$ on the new entity. Then, there exists a +# posterior distribution $q(e_{S^{\prime}}|e_{S})$ over outcomes of unperformed experiments $S^{\prime}$, conditioned on the current +# state. +# +# Furthermore, there also exists a posterior distribution over the unobserved target variable $q(y|e_{S})$. The information +# value of the current state, derived from experimental evidence, can be defined as any statistical or information-theoretic +# measure applied to $q(y|e_{S})$. This can include variance or entropy (among others). The information value +# of the set of experiments performed in $S$ is analogous to $v_{S}$ defined in [static experimental designs](SimpleStatic.md). + +# ### Similarity Weighting + +# There may be many ways to define these posterior distributions, but our approach uses distance-based similarity +# scores to construct weights $w_{j}$ over historical entities which are similar to the new entity. These weights can be used to +# weight the values of $y$ or features associated with $e_{S^{\prime}}$ to construct approximations of $q(y|e_{S})$ and +# $q(e_{S^{\prime}}|e_{S})$. +# +# If there is no evidence associated with a new entity, we assign $w_{j}$ according to some prior distribution (uniform by default). +# Otherwise we use some particular distance and similarity metrics. +# +# For each feature $x\in X$, we consider a function $\rho_x$, which measures the distance between two outputs. Please be aware that the "distances" discussed here do not conform to the mathematical definition of "metrics", even though they are functions derived from underlying metrics (i.e., a square of a metric). This is justified when considering how these "distances" are subsequently converted into probabilistic weights. +# +# By default, we consider the following distances: +# - Rescaled Kronecker delta (i.e., $\rho(x, y)=0$ only when $x=y$, and $\rho(x, y)= \lambda$ under any other circumstances, with $\lambda > 0$) for discrete features (i.e., features whose types are modeled as `MultiClass` type in [ScientificTypes.jl](https://github.com/JuliaAI/ScientificTypes.jl)); +# - Rescaled ("standardized", by default) squared distance $\rho(x, y) = λ \frac{(x - y)^2}{\sigma^2}$, where $\sigma^2$ is the variance of the feature column, estimated with respect to the prior for continuous features. +# - Squared Mahalanobis distance $\rho(x,y) = (x-y)^{⊤}\Sigma^{-1}(x-y)$, where $\Sigma$ is the empirical covariance matrix of the historical data. +# +# For distance metrics assuming independence of features (Kronecker delta and squared distance), given the new entity's experimental state with readouts over the feature set $F = \bigcup X_{e}$, where $e \in S$, we can calculate +# the distance from the $j$-th historical entity as $d_j = \sum_{x\in F} \rho_x (\hat x, x_j)$, where $\hat x$ and $x_j$ denote the readout +# for feature $x$ for the entity being tested and the entity recorded in $j$-th column. +# The squared Mahalanobis distance directly takes in "rows", $\rho(\hat{x},x_{j})$. +# +# Next, we convert distances $d_j$ into probabilistic weights $w_j$. By default, we use a rescaled exponential function, i.e., +# we put $w_j = \exp(-\lambda d_j)$ with $\lambda=0.5$. Notably, $\lambda$'s value determines how belief is distributed across the historical entities. +# Larger values of $\lambda$ concentrate the belief tightly around the "closest" historical entities, while smaller values distribute more belief to more distant entities. +# +# Note that by choosing the squared Mahalanobis distance and the exponential function with a factor of $\lambda=0.5$, the resulting weight effectively equals the density of a [multivariate normal distribution](https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Density_function) fitted to the historical data. A similar assertion applies when we use the sum of "standardized" squared distances instead. However, in this latter case, we "enforce" the independence of the features. +# +# The proper choice of distance and similarity metrics depends on insight into the dataset at hand. Weights can then be used to construct +# weighted histograms or density estimators for the posterior distributions of interest, or to directly resample historical rows. + +# ![weights](assets/generative_weights.png) + +# ### Policy Search + +# Given these parts, searching for optimal experimental designs (actions arranged in an efficient way) depends on what our objective sense is. +# +# - The search may continue until the uncertainty about the posterior distribution of the target variable falls below a certain level. Our aim is to minimize the anticipated combined monetary cost and execution time of the search (considered as a "negative" reward). If all experiments are conducted without reaching below the required uncertainty level, or if the maximum number of experiments is exceeded, we penalize this scenario with an "infinitely negative" reward. +# - We may aim to minimize the expected uncertainty while being constrained by the combined costs of the experiment. +# - Alternatively, we could maximize the value of experimental evidence, adjusted for the incurred experimental costs. + +# Standard Markov decision process (MDP) algorithms can be used to solve this problem (offline learning) or construct the policy (online learning) for the sequential decision-making. + +# Our MDP's state space is finite-dimensional but generally continuous due to the allowance of continuous features, which complicates the problem and few algorithms specialize in this area. + +# We used the Double Progressive Widening Algorithm for this task as detailed in [A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation](https://arxiv.org/abs/1405.5498). + +# In a nutshell, the Double Progressive Widening (DPW) algorithm is designed for online learning in complex environments, particularly those known as Continuous Finite-dimensional Markov Decision Processes where the state space is continuous. The key idea behind DPW is to progressively expand the search tree during the Monte Carlo Tree Search (MCTS) process. The algorithm does so by dynamically and selectively adding states and actions to the tree based on defined heuristics. + +# In the context of online learning, this algorithm addresses the complexity and challenges of real-time decision-making in domains with a large or infinite number of potential actions. As information is gathered in actual runtime, the algorithm explores and exploits this information to make optimal or near-optimal decisions. In other words, DPW permits the learning process to adapt on-the-fly as more data is made available, making it an effective tool for the dynamic and uncertain nature of online environments. + +# In particular, at the core of the Double Progressive Widening (DPW) algorithm are several key components, including expansion, search, and rollout. + +# The search component is where the algorithm sifts through the search tree to determine the most promising actions or states to explore next. By using exploration-exploitation strategies, it can effectively balance its efforts between investigating previously successful actions and venturing into unexplored territories. + +# The expansion phase is where the algorithm grows the search tree by adding new nodes, representing new state-action pairs, to the tree. This is done based on a predefined rule that dictates when and how much the tree should be expanded. This allows the algorithm to methodically discover and consider new potential actions without becoming overwhelmed with choices. + +# Lastly, the rollout stage, also known as a simulation stage, is where the algorithm plays out a series of actions to the end of a game or scenario using a specific policy (like a random policy). The results of these rollouts are then used to update the estimates of the values of state-action pairs, increasing the accuracy of future decisions. + +# ![One iteration of the MCTS algorithm, taken from https://ieeexplore.ieee.org/document/6145622](assets/mcts.png) + +# In the above figure, nodes represent states of the decision process, while edges correspond to actions connecting these states. + +# A graphical summary of a single step of the overall search process to find the next best action, using our running example where a new entity +# has had $e_{1}$ performed out of 3 possible experiments is below: + +# ![search](assets/generative_search.png) + +# ## Synthetic Data Example with Continuous $y$ + +# We now present an example of finding cost-efficient generative designs on synthetic data using the CEEDesigns.jl package. +# +# First we load necessary packages. + +using CEEDesigns, CEEDesigns.GenerativeDesigns +using DataFrames +using ScientificTypes +import Distributions +using Copulas +using POMDPs, POMDPTools, MCTS +using Plots, StatsPlots, StatsBase + +# ### Synthetic Data + +# We use the "Friedman #3" method to make synthetic data for a regression problem from [scikit-learn](https://scikit-learn.org/stable/datasets/sample_generators.html#generators-for-regression). +# It considers $m=4$ features which predict a continuous $y$ via some nonlinear transformations. The marginal +# distributions of each feature are given by the scikit-learn documentation. We additionally use a Gaussian copula +# to induce a specified correlation structure on the features to illustrate the difference between Euclidean and Mahalanobis +# distance metrics. We sample $l=1000$ rows of the historical data. + +make_friedman3 = function (U, noise = 0, friedman3 = true) + size(U, 2) == 4 || error("input U must have 4 columns, has $(size(U,2))") + n = size(U, 1) + X = DataFrame(zeros(Float64, n, 4), :auto) + for i = 1:4 + X[:, i] .= U[:, i] + end + ϵ = noise > 0 ? rand(Distributions.Normal(0, noise), size(X, 1)) : 0 + if friedman3 + X.y = @. atan((X[:, 2] * X[:, 3] - 1 / (X[:, 2] * X[:, 4])) / X[:, 1]) + ϵ + else + ## Friedman #2 + X.y = @. (X[:, 1]^2 + (X[:, 2] * X[:, 3] - 1 / (X[:, 2] * X[:, 4]))^2)^0.5 + ϵ + end + return X +end + +ρ12, ρ13, ρ14, ρ23, ρ24, ρ34 = 0.8, 0.5, 0.3, 0.5, 0.25, 0.4 +Σ = [ + 1 ρ12 ρ13 ρ14 + ρ12 1 ρ23 ρ24 + ρ13 ρ23 1 ρ34 + ρ14 ρ24 ρ34 1 +] + +X1 = Distributions.Uniform(0, 100) +X2 = Distributions.Uniform(40 * π, 560 * π) +X3 = Distributions.Uniform(0, 1) +X4 = Distributions.Uniform(1, 11) + +C = GaussianCopula(Σ) +D = SklarDist(C, (X1, X2, X3, X4)) + +X = rand(D, 1000) + +data = make_friedman3(transpose(X), 0.01) + +data[1:10, :] + +# We can check that the empirical correlation is roughly the same as the specified theoretical values: + +cor(Matrix(data[:, Not(:y)])) + +# We ensure that our algorithms know that we have provided data of specified types. + +types = Dict( + :x1 => ScientificTypes.Continuous, + :x2 => ScientificTypes.Continuous, + :x3 => ScientificTypes.Continuous, + :x4 => ScientificTypes.Continuous, + :y => ScientificTypes.Continuous, +) +data = coerce(data, types); + +# We may plot each feature $x_{i} \in X = {x_{1},x_{2},x_{3},x_{4}}$ against $y$. + +p1 = scatter(data.x1, data.y; legend = false, xlab = "x1") +p2 = scatter(data.x2, data.y; legend = false, xlab = "x2") +p3 = scatter(data.x3, data.y; legend = false, xlab = "x3") +p4 = scatter(data.x4, data.y; legend = false, xlab = "x4") +plot(p1, p2, p3, p4; layout = (2, 2), legend = false) + +# ### Distance-based Similarity + +# Given historical data, a target variable $y$, and metric to quantify uncertainty around +# the posterior distribution on the target $q(y|e_{S})$, the function `DistanceBased` +# returns three functions needed by CEEDesigns.jl: +# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +# - `uncertainty`: this is a function of `evidence`, +# - `weights`: this represents a function of `evidence` that generates probability weights $w_j$ to each row in the historical data. +# +# By default, Euclidean distance is used as the distance metric. In the second +# call to `DistanceBased`, we instead use the squared Mahalanobis distance. +# It is possible to specify different distance metrics for each feature, see our +# [heart disease generative modeling](GenerativeDesigns.md) tutorial for more information. +# In both cases, the squared exponential function is used to convert distances +# to weights. + +(; sampler, uncertainty, weights) = DistanceBased( + data; + target = "y", + uncertainty = Variance(), + similarity = GenerativeDesigns.Exponential(; λ = 5), +); + +(sampler_mh, uncertainty_mh, weights_mh) = DistanceBased( + data; + target = "y", + uncertainty = Variance(), + similarity = GenerativeDesigns.Exponential(; λ = 5), + distance = SquaredMahalanobisDistance(; diagonal = 0), +); + +# We can look at the uncertainty in $y$ for a state where a single +# feature is "observed" at its mean value. Note that uncertainty is generally lower for the squared Mahalanobis distance. +# As we consider only a single non-missing entry, this makes sense as the implemented variant of the squared Mahalanobis distance for missing values effectively multiplies +# the other, quadratic distance, by a factor greater than one. For more details, refer to page 285 of +# [Multivariate outlier detection in incomplete survey data: The epidemic algorithm and transformed rank correlations](https://www.jstor.org/stable/3559861). + +data_uncertainties = + [i => uncertainty(Evidence(i => mean(data[:, i]))) for i in names(data)[1:4]] +sort!(data_uncertainties; by = x -> x[2], rev = true) + +data_uncertainties_mh = + [i => uncertainty_mh(Evidence(i => mean(data[:, i]))) for i in names(data)[1:4]] +sort!(data_uncertainties_mh; by = x -> x[2], rev = true) + +p1 = sticks( + eachindex(data_uncertainties), + [i[2] for i in data_uncertainties]; + xformatter = i -> data_uncertainties[Int(i)][1], + label = false, + title = "Uncertainty\n(Euclidean distance)", +) +p2 = sticks( + eachindex(data_uncertainties_mh), + [i[2] for i in data_uncertainties_mh]; + xformatter = i -> data_uncertainties_mh[Int(i)][1], + label = false, + title = "Uncertainty\n(Mahalanobis distance)", +) + +plot(p1, p2; layout = (1, 2), legend = false) + +# We can view the posterior distribution $q(y|e_{S})$ conditioned on a state (here arbitrarily set to $S = e_{3}$, giving evidence for $x_{3}$). + +evidence = Evidence("x3" => mean(data.x3)) +plot_weights = StatsBase.weights(weights(evidence)) +plot_weights_mh = StatsBase.weights(weights_mh(evidence)) + +p1 = Plots.histogram( + data.y; + weights = plot_weights, + legend = false, + ylabel = "Density", + title = "q(y|eₛ)\n(Euclidean distance)", +) +p2 = Plots.histogram( + data.y; + weights = plot_weights_mh, + legend = false, + ylabel = "Density", + title = "q(y|eₛ)\n(Mahalanobis distance)", +) + +plot(p1, p2; layout = (1, 2), legend = false) + +# Like static designs, generative designs need to be provided a `DataFrame` assigning to each experiment +# a tuple of monetary and time costs $(m_{e},t_{e})$, and what features each experiment provides observations of. +# We'll set up the experimental costs such that experiments which have less marginal uncertainty are more costly +# We finally add a very expensive "final" experiment which can directly observe the target variable. + +observables_experiments = Dict(["x$i" => "e$i" for i = 1:4]) +experiments_costs = Dict([ + observables_experiments[e[1]] => (i, i) => [e[1]] for + (i, e) in enumerate(data_uncertainties_mh) +]) + +experiments_costs["ey"] = (100, 100) => ["y"] + +experiments_costs_df = + DataFrame(; experiment = String[], time = Int[], cost = Int[], measurement = String[]); +push!( + experiments_costs_df, + [ + [ + e, + experiments_costs[e][1][1], + experiments_costs[e][1][2], + only(experiments_costs[e][2]), + ] for e in keys(experiments_costs) + ]..., +); +experiments_costs_df + +# ### Find Cost-efficient Experimental Designs + +# We can now find cost-efficient experimental designs for a new entity that has no measurements (`Evidence()`). +# Our objective sense is to minimize expected experimental combined cost while trying to reduce uncertainty to +# a threshold value. We examine 7 different threshold levels of uncertainty, evenly spaced between 0 and 1, inclusive. +# Additionally we set the `costs_tradeoff` such that equal weight is given to time and monetary cost when +# constructing the combined costs of experiments. +# +# Internally, for each choice of the uncertainty threshold, an instance of a Markov decision problem in [POMDPs.jl](https://github.com/JuliaPOMDP/POMDPs.jl) +# is created, and the `POMDPs.solve` is called on the problem. +# Afterwards, a number of simulations of the decision-making problem is run, all starting with the experimental `state`. +# +# Note that we use the Euclidean distance, due to somewhat faster runtime. + +n_thresholds = 7 +evidence = Evidence() +solver = GenerativeDesigns.DPWSolver(; n_iterations = 500, tree_in_info = true) +repetitions = 5 +mdp_options = (; + max_parallel = length(experiments_costs), + discount = 1.0, + costs_tradeoff = (0.5, 0.5), +) + +designs = efficient_designs( + experiments_costs; + sampler = sampler, + uncertainty = uncertainty, + thresholds = n_thresholds, + evidence = evidence, + solver = solver, + repetitions = repetitions, + mdp_options = mdp_options, +); + +# We plot the Pareto-efficient actions. + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") + +# ## Synthetic Data Example with Discrete $y$ + +# ### Synthetic Data + +# We use n-class classification problem generator from [scikit-learn](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html#sklearn.datasets.make_classification) +# We used parameters given below, for a total of $m=5$ features, with 4 of those informative and 1 redundant (linear combination of the other 4) feature. +# The $y$ has 2 classes, with some added noise. We again sample $l=1000$ rows of historical entities. +# +# The dataset can be approximately reproduced as below: + +## using PyCall +## sklearn = PyCall.pyimport_conda("sklearn", "scikit-learn") +## py""" +## import sklearn as sk +## from sklearn.datasets import make_classification +## """ +## X, y = py"make_classification(n_samples=1000,n_features=5,n_informative=4,n_redundant=1,n_repeated=0,n_classes=2,n_clusters_per_class=2,flip_y=0.1)" +## using DataFrames, CSV +## dat = DataFrame(X, :auto) +## dat.y = y +## CSV.write("./class.csv",dat) + +using CSV + +data = CSV.read("./data/class.csv", DataFrame) + +types = Dict( + :x1 => ScientificTypes.Continuous, + :x2 => ScientificTypes.Continuous, + :x3 => ScientificTypes.Continuous, + :x4 => ScientificTypes.Continuous, + :y => ScientificTypes.Multiclass, +) +data = coerce(data, types); + +# ### Distance-based Similarity + +# We now set up the distance based similarity functions. We use `Entropy` as our metric of uncertainty this time, +# which is more suitable for discrete $y$. + +(; sampler, uncertainty, weights) = DistanceBased( + data; + target = "y", + uncertainty = Entropy(), + similarity = GenerativeDesigns.Exponential(; λ = 5), +); + +# We may also look at the uncertainty from each marginal distribution of features. +# This is a bit nonsensical as the data generating function will create multimodal clusters +# so it will look artifically as if nothing is informative, but that is not the case. + +data_uncertainties = + [i => uncertainty(Evidence(i => mode(data[:, i]))) for i in names(data)[1:end-1]] +sort!(data_uncertainties; by = x -> x[2], rev = true) + +sticks( + eachindex(data_uncertainties), + [i[2] for i in data_uncertainties]; + xformatter = i -> data_uncertainties[Int(i)][1], + label = false, + ylab = "Uncertainty", +) + +# We can view the posterior distribution $q(y|e_{S})$ when we consider the state as evidence a single measurement +# of the first feature, set to the mean of that distribution. + +evidence = Evidence("x1" => mean(data.x1)) +plot_weights = StatsBase.weights(weights(evidence)) + +target_belief = countmap(data[!, "y"], plot_weights) +p = bar( + 0:1, + [target_belief[0], target_belief[1]]; + xrot = 40, + ylabel = "probability", + title = "unc: $(round(uncertainty(evidence), digits=1))", + kind = :bar, + legend = false, +); +xticks!(p, 0:1, ["0", "1"]); +p + +# Like previous examples, we'll set up the experimental costs such that experiments which have less marginal uncertainty +# are more costly, add a final very expensive experiment directly on the target variable. + +observables_experiments = Dict(["x$i" => "e$i" for i = 1:5]) +experiments_costs = Dict([ + observables_experiments[e[1]] => (i, i) => [e[1]] for + (i, e) in enumerate(sort(data_uncertainties; by = x -> x[2], rev = true)) +]) + +experiments_costs["ey"] = (100, 100) => ["y"] + +experiments_costs_df = + DataFrame(; experiment = String[], time = Int[], cost = Int[], measurement = String[]); +push!( + experiments_costs_df, + [ + [ + e, + experiments_costs[e][1][1], + experiments_costs[e][1][2], + only(experiments_costs[e][2]), + ] for e in keys(experiments_costs) + ]..., +); +experiments_costs_df + +# ### Find Cost-efficient Experimental Designs + +# We can now find sets of cost-efficient experimental designs for a new entity with no measurements (`Evidence()`). +# We use the same solver parameters as for the exaple with continuous $y$, and plot the resulting +# Pareto front. + +n_thresholds = 7 +evidence = Evidence() +solver = GenerativeDesigns.DPWSolver(; n_iterations = 500, tree_in_info = true) +repetitions = 5 +mdp_options = (; + max_parallel = length(experiments_costs), + discount = 1.0, + costs_tradeoff = (0.5, 0.5), +) + +designs = efficient_designs( + experiments_costs; + sampler = sampler, + uncertainty = uncertainty, + thresholds = n_thresholds, + evidence = evidence, + solver = solver, + repetitions = repetitions, + mdp_options = mdp_options, +); + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") diff --git a/docs/src/tutorials/SimpleGenerative.md b/docs/src/tutorials/SimpleGenerative.md new file mode 100644 index 0000000..19c6839 --- /dev/null +++ b/docs/src/tutorials/SimpleGenerative.md @@ -0,0 +1,556 @@ +```@meta +EditURL = "SimpleGenerative.jl" +``` + +# Generative Experimental Designs + +This document describes the theoretical background behind tools in CEEDesigns.jl for generative experimental designs +and demonstrates uses on synthetic data examples. + +## Setting + +Generative experimental designs differ from static designs in that the experimental design is specific for (or "personalized") to an incoming entity. +Personalized cost-efficient experimental designs for the new entity are made based on existing information (if any) about the new entity, +and from posterior distributions of unobserved features of that entity conditional on its observed features and historical data. +The entities may be patients, molecular compounds, or any other objects where one has a store of historical data +and seeks to find efficient experimental designs to learn more about a new arrival. + +The personalized experimental designs are motivated by the fact that the value of information collected from an experiment +often differs across subsets of the population of entities. + +![information value matrix](assets/information_value_matrix.png) + +In the context of static designs, we do not aspire to capture variation in information gain across different entities. Instead, we assume all entities come from a "Population" with a uniform information gain, in which case "Experiment C" would provide the maximum information value. +On the other hand, if we have the ability to discern if the entity belong to subpopulations "Population 1" or "Population 2," then we can tailor our +design to suggest either "Experiment A" or "Experiment B." Clearly, in the limit of a maximally heterogenous population, each +entity has its own "row." Our tools are able to handle the entire spectrum of such scenarios though distance based similarity, +described below. + +## Theoretical Framework + +### Historical Data + +Like static designs, we consider a set $E$ of $n$ experiments, each with an associated tuple $(m_{e},t_{e})$ of monetary and +time costs (for more details on experiments and arrangements, see the tutorial on [static experimental designs](SimpleStatic.md)). + +Additionally, consider a historical dataset giving measurements on $m$ features $X = \{x_1, \ldots, x_m\}$ for $l$ entities +(with entities and features representing rows and columns, respectively). Each experiment $e$ may yield measurements on some +subset of features $(X_{e}\subseteq X)$. + +Furthermore there is an additional column $y$ which is a target variable we want to predict (CEEDesigns.jl may allow $y$ +to be a vector, but we assume it is scalar here for ease of presentation). + +Letting $m=3$, then the historical dataset may be visualized as the following table: + +![historical data](assets/generative_historical.png) + +### New Entity + +When a new entity arrives, it will have an unknown outcome $y$ and unknown values of some or all features $x_{i} \in X$. +We call the _state_ of the new entity the set of experiments conducted upon it so far (if any), along with the +measurements on any features produced by those experiments (if any), called _evidence_. + +Consider the case where there are $n=3$ experiments, each of which yields a measurement on a single feature. Then +the state of a new entity arriving for which $e_{1}$ has already been performed will look like: + +![state](assets/generative_state.png) + +Letting $S$ denote the set of experiments which have been performed so far, then $S^{\prime}=E\S$ are unperformed experiments. +Then, _actions_ one can perform to learn more about the new entity are subsets of $S^{\prime}$. The size of subsets +is limited by the maximum number of parallel experiments. + +In our running example, if the maximum number of parallel experiments is at least 2, then the available actions are: + +![actions](assets/generative_actions.png) + +### Posterior Distributions + +Let $e_{S}$ be a random variable denoting the outcome of some experiments $S$ on the new entity. Then, there exists a +posterior distribution $q(e_{S^{\prime}}|e_{S})$ over outcomes of unperformed experiments $S^{\prime}$, conditioned on the current +state. + +Furthermore, there also exists a posterior distribution over the unobserved target variable $q(y|e_{S})$. The information +value of the current state, derived from experimental evidence, can be defined as any statistical or information-theoretic +measure applied to $q(y|e_{S})$. This can include variance or entropy (among others). The information value +of the set of experiments performed in $S$ is analogous to $v_{S}$ defined in [static experimental designs](SimpleStatic.md). + +### Similarity Weighting + +There may be many ways to define these posterior distributions, but our approach uses distance-based similarity +scores to construct weights $w_{j}$ over historical entities which are similar to the new entity. These weights can be used to +weight the values of $y$ or features associated with $e_{S^{\prime}}$ to construct approximations of $q(y|e_{S})$ and +$q(e_{S^{\prime}}|e_{S})$. + +If there is no evidence associated with a new entity, we assign $w_{j}$ according to some prior distribution (uniform by default). +Otherwise we use some particular distance and similarity metrics. + +For each feature $x\in X$, we consider a function $\rho_x$, which measures the distance between two outputs. Please be aware that the "distances" discussed here do not conform to the mathematical definition of "metrics", even though they are functions derived from underlying metrics (i.e., a square of a metric). This is justified when considering how these "distances" are subsequently converted into probabilistic weights. + +By default, we consider the following distances: +- Rescaled Kronecker delta (i.e., $\rho(x, y)=0$ only when $x=y$, and $\rho(x, y)= \lambda$ under any other circumstances, with $\lambda > 0$) for discrete features (i.e., features whose types are modeled as `MultiClass` type in [ScientificTypes.jl](https://github.com/JuliaAI/ScientificTypes.jl)); +- Rescaled ("standardized", by default) squared distance $\rho(x, y) = λ \frac{(x - y)^2}{\sigma^2}$, where $\sigma^2$ is the variance of the feature column, estimated with respect to the prior for continuous features. +- Squared Mahalanobis distance $\rho(x,y) = (x-y)^{⊤}\Sigma^{-1}(x-y)$, where $\Sigma$ is the empirical covariance matrix of the historical data. + +For distance metrics assuming independence of features (Kronecker delta and squared distance), given the new entity's experimental state with readouts over the feature set $F = \bigcup X_{e}$, where $e \in S$, we can calculate +the distance from the $j$-th historical entity as $d_j = \sum_{x\in F} \rho_x (\hat x, x_j)$, where $\hat x$ and $x_j$ denote the readout +for feature $x$ for the entity being tested and the entity recorded in $j$-th column. +The squared Mahalanobis distance directly takes in "rows", $\rho(\hat{x},x_{j})$. + +Next, we convert distances $d_j$ into probabilistic weights $w_j$. By default, we use a rescaled exponential function, i.e., +we put $w_j = \exp(-\lambda d_j)$ with $\lambda=0.5$. Notably, $\lambda$'s value determines how belief is distributed across the historical entities. +Larger values of $\lambda$ concentrate the belief tightly around the "closest" historical entities, while smaller values distribute more belief to more distant entities. + +Note that by choosing the squared Mahalanobis distance and the exponential function with a factor of $\lambda=0.5$, the resulting weight effectively equals the density of a [multivariate normal distribution](https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Density_function) fitted to the historical data. A similar assertion applies when we use the sum of "standardized" squared distances instead. However, in this latter case, we "enforce" the independence of the features. + +The proper choice of distance and similarity metrics depends on insight into the dataset at hand. Weights can then be used to construct +weighted histograms or density estimators for the posterior distributions of interest, or to directly resample historical rows. + +![weights](assets/generative_weights.png) + +### Policy Search + +Given these parts, searching for optimal experimental designs (actions arranged in an efficient way) depends on what our objective sense is. + +- The search may continue until the uncertainty about the posterior distribution of the target variable falls below a certain level. Our aim is to minimize the anticipated combined monetary cost and execution time of the search (considered as a "negative" reward). If all experiments are conducted without reaching below the required uncertainty level, or if the maximum number of experiments is exceeded, we penalize this scenario with an "infinitely negative" reward. +- We may aim to minimize the expected uncertainty while being constrained by the combined costs of the experiment. +- Alternatively, we could maximize the value of experimental evidence, adjusted for the incurred experimental costs. + +Standard Markov decision process (MDP) algorithms can be used to solve this problem (offline learning) or construct the policy (online learning) for the sequential decision-making. + +Our MDP's state space is finite-dimensional but generally continuous due to the allowance of continuous features, which complicates the problem and few algorithms specialize in this area. + +We used the Double Progressive Widening Algorithm for this task as detailed in [A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation](https://arxiv.org/abs/1405.5498). + +In a nutshell, the Double Progressive Widening (DPW) algorithm is designed for online learning in complex environments, particularly those known as Continuous Finite-dimensional Markov Decision Processes where the state space is continuous. The key idea behind DPW is to progressively expand the search tree during the Monte Carlo Tree Search (MCTS) process. The algorithm does so by dynamically and selectively adding states and actions to the tree based on defined heuristics. + +In the context of online learning, this algorithm addresses the complexity and challenges of real-time decision-making in domains with a large or infinite number of potential actions. As information is gathered in actual runtime, the algorithm explores and exploits this information to make optimal or near-optimal decisions. In other words, DPW permits the learning process to adapt on-the-fly as more data is made available, making it an effective tool for the dynamic and uncertain nature of online environments. + +In particular, at the core of the Double Progressive Widening (DPW) algorithm are several key components, including expansion, search, and rollout. + +The search component is where the algorithm sifts through the search tree to determine the most promising actions or states to explore next. By using exploration-exploitation strategies, it can effectively balance its efforts between investigating previously successful actions and venturing into unexplored territories. + +The expansion phase is where the algorithm grows the search tree by adding new nodes, representing new state-action pairs, to the tree. This is done based on a predefined rule that dictates when and how much the tree should be expanded. This allows the algorithm to methodically discover and consider new potential actions without becoming overwhelmed with choices. + +Lastly, the rollout stage, also known as a simulation stage, is where the algorithm plays out a series of actions to the end of a game or scenario using a specific policy (like a random policy). The results of these rollouts are then used to update the estimates of the values of state-action pairs, increasing the accuracy of future decisions. + +![One iteration of the MCTS algorithm, taken from https://ieeexplore.ieee.org/document/6145622](assets/mcts.png) + +In the above figure, nodes represent states of the decision process, while edges correspond to actions connecting these states. + +A graphical summary of a single step of the overall search process to find the next best action, using our running example where a new entity +has had $e_{1}$ performed out of 3 possible experiments is below: + +![search](assets/generative_search.png) + +## Synthetic Data Example with Continuous $y$ + +We now present an example of finding cost-efficient generative designs on synthetic data using the CEEDesigns.jl package. + +First we load necessary packages. + +````@example SimpleGenerative +using CEEDesigns, CEEDesigns.GenerativeDesigns +using DataFrames +using ScientificTypes +import Distributions +using Copulas +using POMDPs, POMDPTools, MCTS +using Plots, StatsPlots, StatsBase +```` + +### Synthetic Data + +We use the "Friedman #3" method to make synthetic data for a regression problem from [scikit-learn](https://scikit-learn.org/stable/datasets/sample_generators.html#generators-for-regression). +It considers $m=4$ features which predict a continuous $y$ via some nonlinear transformations. The marginal +distributions of each feature are given by the scikit-learn documentation. We additionally use a Gaussian copula +to induce a specified correlation structure on the features to illustrate the difference between Euclidean and Mahalanobis +distance metrics. We sample $l=1000$ rows of the historical data. + +````@example SimpleGenerative +make_friedman3 = function (U, noise = 0, friedman3 = true) + size(U, 2) == 4 || error("input U must have 4 columns, has $(size(U,2))") + n = size(U, 1) + X = DataFrame(zeros(Float64, n, 4), :auto) + for i = 1:4 + X[:, i] .= U[:, i] + end + ϵ = noise > 0 ? rand(Distributions.Normal(0, noise), size(X, 1)) : 0 + if friedman3 + X.y = @. atan((X[:, 2] * X[:, 3] - 1 / (X[:, 2] * X[:, 4])) / X[:, 1]) + ϵ + else + # Friedman #2 + X.y = @. (X[:, 1]^2 + (X[:, 2] * X[:, 3] - 1 / (X[:, 2] * X[:, 4]))^2)^0.5 + ϵ + end + return X +end + +ρ12, ρ13, ρ14, ρ23, ρ24, ρ34 = 0.8, 0.5, 0.3, 0.5, 0.25, 0.4 +Σ = [ + 1 ρ12 ρ13 ρ14 + ρ12 1 ρ23 ρ24 + ρ13 ρ23 1 ρ34 + ρ14 ρ24 ρ34 1 +] + +X1 = Distributions.Uniform(0, 100) +X2 = Distributions.Uniform(40 * π, 560 * π) +X3 = Distributions.Uniform(0, 1) +X4 = Distributions.Uniform(1, 11) + +C = GaussianCopula(Σ) +D = SklarDist(C, (X1, X2, X3, X4)) + +X = rand(D, 1000) + +data = make_friedman3(transpose(X), 0.01) + +data[1:10, :] +```` + +We can check that the empirical correlation is roughly the same as the specified theoretical values: + +````@example SimpleGenerative +cor(Matrix(data[:, Not(:y)])) +```` + +We ensure that our algorithms know that we have provided data of specified types. + +````@example SimpleGenerative +types = Dict( + :x1 => ScientificTypes.Continuous, + :x2 => ScientificTypes.Continuous, + :x3 => ScientificTypes.Continuous, + :x4 => ScientificTypes.Continuous, + :y => ScientificTypes.Continuous, +) +data = coerce(data, types); +nothing #hide +```` + +We may plot each feature $x_{i} \in X = {x_{1},x_{2},x_{3},x_{4}}$ against $y$. + +````@example SimpleGenerative +p1 = scatter(data.x1, data.y; legend = false, xlab = "x1") +p2 = scatter(data.x2, data.y; legend = false, xlab = "x2") +p3 = scatter(data.x3, data.y; legend = false, xlab = "x3") +p4 = scatter(data.x4, data.y; legend = false, xlab = "x4") +plot(p1, p2, p3, p4; layout = (2, 2), legend = false) +```` + +### Distance-based Similarity + +Given historical data, a target variable $y$, and metric to quantify uncertainty around +the posterior distribution on the target $q(y|e_{S})$, the function `DistanceBased` +returns three functions needed by CEEDesigns.jl: +- `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +- `uncertainty`: this is a function of `evidence`, +- `weights`: this represents a function of `evidence` that generates probability weights $w_j$ to each row in the historical data. + +By default, Euclidean distance is used as the distance metric. In the second +call to `DistanceBased`, we instead use the squared Mahalanobis distance. +It is possible to specify different distance metrics for each feature, see our +[heart disease generative modeling](GenerativeDesigns.md) tutorial for more information. +In both cases, the squared exponential function is used to convert distances +to weights. + +````@example SimpleGenerative +(; sampler, uncertainty, weights) = DistanceBased( + data; + target = "y", + uncertainty = Variance(), + similarity = GenerativeDesigns.Exponential(; λ = 5), +); + +(sampler_mh, uncertainty_mh, weights_mh) = DistanceBased( + data; + target = "y", + uncertainty = Variance(), + similarity = GenerativeDesigns.Exponential(; λ = 5), + distance = SquaredMahalanobisDistance(; diagonal = 0), +); +nothing #hide +```` + +We can look at the uncertainty in $y$ for a state where a single +feature is "observed" at its mean value. Note that uncertainty is generally lower for the squared Mahalanobis distance. +As we consider only a single non-missing entry, this makes sense as the implemented variant of the squared Mahalanobis distance for missing values effectively multiplies +the other, quadratic distance, by a factor greater than one. For more details, refer to page 285 of +[Multivariate outlier detection in incomplete survey data: The epidemic algorithm and transformed rank correlations](https://www.jstor.org/stable/3559861). + +````@example SimpleGenerative +data_uncertainties = + [i => uncertainty(Evidence(i => mean(data[:, i]))) for i in names(data)[1:4]] +sort!(data_uncertainties; by = x -> x[2], rev = true) + +data_uncertainties_mh = + [i => uncertainty_mh(Evidence(i => mean(data[:, i]))) for i in names(data)[1:4]] +sort!(data_uncertainties_mh; by = x -> x[2], rev = true) + +p1 = sticks( + eachindex(data_uncertainties), + [i[2] for i in data_uncertainties]; + xformatter = i -> data_uncertainties[Int(i)][1], + label = false, + title = "Uncertainty\n(Euclidean distance)", +) +p2 = sticks( + eachindex(data_uncertainties_mh), + [i[2] for i in data_uncertainties_mh]; + xformatter = i -> data_uncertainties_mh[Int(i)][1], + label = false, + title = "Uncertainty\n(Mahalanobis distance)", +) + +plot(p1, p2; layout = (1, 2), legend = false) +```` + +We can view the posterior distribution $q(y|e_{S})$ conditioned on a state (here arbitrarily set to $S = e_{3}$, giving evidence for $x_{3}$). + +````@example SimpleGenerative +evidence = Evidence("x3" => mean(data.x3)) +plot_weights = StatsBase.weights(weights(evidence)) +plot_weights_mh = StatsBase.weights(weights_mh(evidence)) + +p1 = Plots.histogram( + data.y; + weights = plot_weights, + legend = false, + ylabel = "Density", + title = "q(y|eₛ)\n(Euclidean distance)", +) +p2 = Plots.histogram( + data.y; + weights = plot_weights_mh, + legend = false, + ylabel = "Density", + title = "q(y|eₛ)\n(Mahalanobis distance)", +) + +plot(p1, p2; layout = (1, 2), legend = false) +```` + +Like static designs, generative designs need to be provided a `DataFrame` assigning to each experiment +a tuple of monetary and time costs $(m_{e},t_{e})$, and what features each experiment provides observations of. +We'll set up the experimental costs such that experiments which have less marginal uncertainty are more costly +We finally add a very expensive "final" experiment which can directly observe the target variable. + +````@example SimpleGenerative +observables_experiments = Dict(["x$i" => "e$i" for i = 1:4]) +experiments_costs = Dict([ + observables_experiments[e[1]] => (i, i) => [e[1]] for + (i, e) in enumerate(data_uncertainties_mh) +]) + +experiments_costs["ey"] = (100, 100) => ["y"] + +experiments_costs_df = + DataFrame(; experiment = String[], time = Int[], cost = Int[], measurement = String[]); +push!( + experiments_costs_df, + [ + [ + e, + experiments_costs[e][1][1], + experiments_costs[e][1][2], + only(experiments_costs[e][2]), + ] for e in keys(experiments_costs) + ]..., +); +experiments_costs_df +```` + +### Find Cost-efficient Experimental Designs + +We can now find cost-efficient experimental designs for a new entity that has no measurements (`Evidence()`). +Our objective sense is to minimize expected experimental combined cost while trying to reduce uncertainty to +a threshold value. We examine 7 different threshold levels of uncertainty, evenly spaced between 0 and 1, inclusive. +Additionally we set the `costs_tradeoff` such that equal weight is given to time and monetary cost when +constructing the combined costs of experiments. + +Internally, for each choice of the uncertainty threshold, an instance of a Markov decision problem in [POMDPs.jl](https://github.com/JuliaPOMDP/POMDPs.jl) +is created, and the `POMDPs.solve` is called on the problem. +Afterwards, a number of simulations of the decision-making problem is run, all starting with the experimental `state`. + +Note that we use the Euclidean distance, due to somewhat faster runtime. + +````@example SimpleGenerative +n_thresholds = 7 +evidence = Evidence() +solver = GenerativeDesigns.DPWSolver(; n_iterations = 500, tree_in_info = true) +repetitions = 5 +mdp_options = (; + max_parallel = length(experiments_costs), + discount = 1.0, + costs_tradeoff = (0.5, 0.5), +) + +designs = efficient_designs( + experiments_costs; + sampler = sampler, + uncertainty = uncertainty, + thresholds = n_thresholds, + evidence = evidence, + solver = solver, + repetitions = repetitions, + mdp_options = mdp_options, +); +nothing #hide +```` + +We plot the Pareto-efficient actions. + +````@example SimpleGenerative +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") +```` + +## Synthetic Data Example with Discrete $y$ + +### Synthetic Data + +We use n-class classification problem generator from [scikit-learn](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html#sklearn.datasets.make_classification) +We used parameters given below, for a total of $m=5$ features, with 4 of those informative and 1 redundant (linear combination of the other 4) feature. +The $y$ has 2 classes, with some added noise. We again sample $l=1000$ rows of historical entities. + +The dataset can be approximately reproduced as below: + +````@example SimpleGenerative +# using PyCall +# sklearn = PyCall.pyimport_conda("sklearn", "scikit-learn") +# py""" +# import sklearn as sk +# from sklearn.datasets import make_classification +# """ +# X, y = py"make_classification(n_samples=1000,n_features=5,n_informative=4,n_redundant=1,n_repeated=0,n_classes=2,n_clusters_per_class=2,flip_y=0.1)" +# using DataFrames, CSV +# dat = DataFrame(X, :auto) +# dat.y = y +# CSV.write("./class.csv",dat) + +using CSV + +data = CSV.read("./data/class.csv", DataFrame) + +types = Dict( + :x1 => ScientificTypes.Continuous, + :x2 => ScientificTypes.Continuous, + :x3 => ScientificTypes.Continuous, + :x4 => ScientificTypes.Continuous, + :y => ScientificTypes.Multiclass, +) +data = coerce(data, types); +nothing #hide +```` + +### Distance-based Similarity + +We now set up the distance based similarity functions. We use `Entropy` as our metric of uncertainty this time, +which is more suitable for discrete $y$. + +````@example SimpleGenerative +(; sampler, uncertainty, weights) = DistanceBased( + data; + target = "y", + uncertainty = Entropy(), + similarity = GenerativeDesigns.Exponential(; λ = 5), +); +nothing #hide +```` + +We may also look at the uncertainty from each marginal distribution of features. +This is a bit nonsensical as the data generating function will create multimodal clusters +so it will look artifically as if nothing is informative, but that is not the case. + +````@example SimpleGenerative +data_uncertainties = + [i => uncertainty(Evidence(i => mode(data[:, i]))) for i in names(data)[1:end-1]] +sort!(data_uncertainties; by = x -> x[2], rev = true) + +sticks( + eachindex(data_uncertainties), + [i[2] for i in data_uncertainties]; + xformatter = i -> data_uncertainties[Int(i)][1], + label = false, + ylab = "Uncertainty", +) +```` + +We can view the posterior distribution $q(y|e_{S})$ when we consider the state as evidence a single measurement +of the first feature, set to the mean of that distribution. + +````@example SimpleGenerative +evidence = Evidence("x1" => mean(data.x1)) +plot_weights = StatsBase.weights(weights(evidence)) + +target_belief = countmap(data[!, "y"], plot_weights) +p = bar( + 0:1, + [target_belief[0], target_belief[1]]; + xrot = 40, + ylabel = "probability", + title = "unc: $(round(uncertainty(evidence), digits=1))", + kind = :bar, + legend = false, +); +xticks!(p, 0:1, ["0", "1"]); +p +```` + +Like previous examples, we'll set up the experimental costs such that experiments which have less marginal uncertainty +are more costly, add a final very expensive experiment directly on the target variable. + +````@example SimpleGenerative +observables_experiments = Dict(["x$i" => "e$i" for i = 1:5]) +experiments_costs = Dict([ + observables_experiments[e[1]] => (i, i) => [e[1]] for + (i, e) in enumerate(sort(data_uncertainties; by = x -> x[2], rev = true)) +]) + +experiments_costs["ey"] = (100, 100) => ["y"] + +experiments_costs_df = + DataFrame(; experiment = String[], time = Int[], cost = Int[], measurement = String[]); +push!( + experiments_costs_df, + [ + [ + e, + experiments_costs[e][1][1], + experiments_costs[e][1][2], + only(experiments_costs[e][2]), + ] for e in keys(experiments_costs) + ]..., +); +experiments_costs_df +```` + +### Find Cost-efficient Experimental Designs + +We can now find sets of cost-efficient experimental designs for a new entity with no measurements (`Evidence()`). +We use the same solver parameters as for the exaple with continuous $y$, and plot the resulting +Pareto front. + +````@example SimpleGenerative +n_thresholds = 7 +evidence = Evidence() +solver = GenerativeDesigns.DPWSolver(; n_iterations = 500, tree_in_info = true) +repetitions = 5 +mdp_options = (; + max_parallel = length(experiments_costs), + discount = 1.0, + costs_tradeoff = (0.5, 0.5), +) + +designs = efficient_designs( + experiments_costs; + sampler = sampler, + uncertainty = uncertainty, + thresholds = n_thresholds, + evidence = evidence, + solver = solver, + repetitions = repetitions, + mdp_options = mdp_options, +); + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") +```` + diff --git a/docs/src/tutorials/SimpleStatic.jl b/docs/src/tutorials/SimpleStatic.jl new file mode 100644 index 0000000..09890e9 --- /dev/null +++ b/docs/src/tutorials/SimpleStatic.jl @@ -0,0 +1,173 @@ +# # Static Experimental Designs + +# In this document we describe the theoretical background behind the tools in CEEDesigns.jl for producing optimal "static experimental designs," i.e., +# arrangements of experiments that exist along a Pareto-optimal tradeoff between cost and information gain. +# We also show an example with synthetic data. + +# ## Setting + +# Consider the following scenario. There exists a set of experiments, each of which, when performed, yields +# measurements on one or more observables (features). Each subset of observables (and therefore each subset of experiments) +# has some "information value," which is intentionally vaguely defined for generality, but for example, may be +# a loss function if that subset is used to train some machine learning model. It is generally the value of acquiring that information. +# Finally, each experiment has some monetary cost and execution time to perform the experiment, and +# the user has some known tradeoff between overall execution time and cost. +# +# CEEDesigns.jl provides tools to take these inputs and produce a set of optimal "arrangements" of experiments for each +# subset of experiments that form a Pareto front along the tradeoff between information gain and total combined cost +# (monetary and time). This allows informed decisions to be made, for example, regarding how to allocate scarce +# resources to a set of experiments that attain some acceptable level of information (or, conversely, reduce +# uncertainty below some level). +# +# The arrangements produced by the tools introduced in this tutorial are called "static" because they inherently +# assume that for each experiment, future data will deterministically yield the same information gain as the "historical" data did. +# This information gain from the "historical" data is quantified based on certain aggregate statistics. +# +# We can also consider "generative experimental designs," where the information gain is modeled as a random variable. This concept is detailed in another [tutorial](./SimpleGenerative.jl). +# +# This tutorial introduces the theoretical framework behind static experimental designs with synthetic data. +# For examples using real data, please see our other tutorials. + +# ## Theoretical Framework + +# ### Experiments + +# Let $E = \{ e_1, \ldots, e_n\}$ be a set of $n$ experiments (i.e., $|E|=n$). Each experiment $e \in E$ has an +# associated tuple $(m_{e},t_{e})$, giving the monetary cost and time duration required to perform experiment $e$. + +# ![experiments](assets/static_experiments.png) + +# Consider $P(E)$, the power set of experiments (i.e., every possible subset of experiments). Each subset of +# experiments $S\in P(E)$ has an associated value $v_{S}$, which is the value of the experiments contained in $S$. +# This may be given by the loss function associated with a prediction task where the information yielded from $S$ +# is used as predictor variables, or some other notion of information value. + +# ![experiments](assets/static_powerset.png) + +# ### Arrangements + +# If experiments within a subset $S$ can be performed simultaneously (in parallel), then each $S$ may be arranged +# optimally with respect to time. An arrangement $O_{S}$ of $S$ is a partition of the experiments in $S$ such that +# the size of each partition is not larger than the maximum number of experiments that may be done in parallel. +# +# Let $l$ be the number of partitions, and $o_{i}$ a partition in $O_{S}$. Then the arrangement is a surjection from $S$ +# onto $O_{S}$. If no experiments can be done in parallel, then $l=|S|$. If all experiments are done in parallel, then +# $l=1$. Other arrangements fall between these extremes. + +# ![experiments](assets/static_arrangement.png) + +# ### Optimal Arrangements + +# To find the optimal arrangement for each $S$ we need to know the cost of $O_{S}$. The monetary cost of $O_{S}$ is simply +# the sum of the costs of each experiment: $m_{O_{S}}=\sum_{e\in S} m_{e}$. +# The total time required is the sum of the maximum time *of each partition*. This is because while each partition in the +# arrangement is done in serial, experiments within partitions are done in parallel. That is, $t_{O_{S}}=\sum_{i=1}^{l} \text{max} \{ t_{e}: e \in o_{i}\}$. +# Given these costs and a parameter $\lambda$ which controls the tradeoff between monetary cost and time, the combined +# cost of an arrangement is: $\lambda m_{O_{S}} + (1-\lambda) t_{O_{S}}$. +# +# For instance, consider the experiments $S = \{e_{1},e_{2},e_{3},e_{4}\}$, with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$. +# If we conduct experiments $e_1$ through $e_4$ in sequence, this would correspond to an arrangement +# $O_{S} = (\{ e_1 \}, \{ e_2 \}, \{ e_3 \}, \{ e_4 \})$ with a total cost of $m_{O_{S}} = 4$ and $t_{O_{S}} = 10$. +# +# However, if we decide to conduct $e_1$ in parallel with $e_3$, and $e_2$ with $e_4$, we would obtain an arrangement +# $O_{S} = (\{ e_1, e_3 \}, \{ e_2, e_4 \})$ with a total cost of $m_{O_{S}} = 4$, and $t_{O_{S}} = 3 + 4 = 7$. +# +# Continuing our example and assuming a maximum of two parallel experiments, the optimal arrangement is to conduct +# $e_1$ in parallel with $e_2$, and $e_3$ with $e_4$. This results in an arrangement $O_{S} = (\{ e_1, e_2 \}, \{ e_3, e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 2 + 4 = 6$. +# +# In fact, it can be readily demonstrated that the optimal arrangement can be found by ordering the experiments in +# S in descending order according to their execution times. Consequently, the experiments are grouped sequentially +# into partitions whose size equals the maximum number of parallel experiments, except possibly for the final set, +# if the maximum number of parallel experiments does not divide $S$ evenly. + +# ## Synthetic Data Example + +# We now present an example of finding cost-efficient designs for synthetic data using the CEEDesigns.jl package. +# +# First we load necessary packages. + +using CEEDesigns, CEEDesigns.StaticDesigns +using Combinatorics: powerset +using DataFrames +using POMDPs, POMDPTools, MCTS + +# We consider a situation where there are 3 experiments, and we draw a value of their "loss function" +# or "entropy" from the uniform distribution on the unit interval for each. +# +# For each $S\in P(E)$, we simulate the information value ($v_{S}$) of $S$ as the product of +# the values for each individual experiment. +# Therefore, because smaller values are better, any subset containing multiple experiments is guaranteed to be +# more "valuable" than any component experiment. + +experiments = ["e1", "e2", "e3"]; +experiments_val = Dict([e => rand() for e in experiments]); + +experiments_evals = Dict(map(Set.(collect(powerset(experiments)))) do s + if length(s) > 0 + s => prod([experiments_val[i] for i in s]) + else + return s => 1.0 + end +end); + +# See our other tutorial on [heart disease triage](StaticDesigns.md) for an example of using CEEDesigns.jl's built-in +# compatability with machine learning models from `MLJ.jl` to evalute performance of experiments using +# predictive accuracy as information value. +# +# Next we set up the costs $(m_{e},t_{e})$ for each experiment. +# Better experiments are more costly, both in terms of time and monetary cost. We print +# the data frame showing each experiment and its associated costs. + +experiments_costs = Dict( + sort(collect(keys(experiments_val)); by = k -> experiments_val[k], rev = true) .=> + tuple.(1:3, 1:3), +); + +DataFrame(; + experiment = collect(keys(experiments_costs)), + time = getindex.(values(experiments_costs), 1), + cost = getindex.(values(experiments_costs), 2), +) + +# We can plot the experiments ordered by their information value. + +plot_evals( + experiments_evals; + f = x -> sort(collect(keys(x)); by = k -> x[k], rev = true), + ylabel = "loss", +) + +# We print the data frame showing each subset of experiments and its overall information value. + +df_values = DataFrame(; + S = collect.(collect(keys(experiments_evals))), + value = collect(values(experiments_evals)), +) + +sort(df_values, order(:value; rev = true)) + +# Now we are ready to find the subsets of experiments giving an optimal tradeoff between information +# value and combined cost. CEED exports a function `efficient_designs` +# which formulates the problem of finding optimal arrangements as a Markov Decision Process and solves +# optimal arrangements for each subset on the Pareto frontier. +# +# We set $\lambda=0.5$, the parameter controlling the relative weight given to monetary versus time costs +# with the tuple `tradeoff`. + +max_parallel = 2; +tradeoff = (0.5, 0.5); + +designs = efficient_designs( + experiments_costs, + experiments_evals; + max_parallel = max_parallel, + tradeoff = tradeoff, +); + +# Finally we may produce a plot of the set of cost-efficient experimental designs. The set of designs +# is plotted along a Pareto frontier giving tradeoff between informatio value (y-axis) and combined cost (x-axis). +# Note that because we set the maximum number of parallel experiments equal to 2, the efficient design for the complete set +# of experiments groups the experiments with long execution times together (see plot legend; each group within a partition is +# prefixed with a number). + +plot_front(designs; labels = make_labels(designs), ylabel = "loss") diff --git a/docs/src/tutorials/SimpleStatic.md b/docs/src/tutorials/SimpleStatic.md new file mode 100644 index 0000000..13ab74c --- /dev/null +++ b/docs/src/tutorials/SimpleStatic.md @@ -0,0 +1,194 @@ +```@meta +EditURL = "SimpleStatic.jl" +``` + +# Static Experimental Designs + +In this document we describe the theoretical background behind the tools in CEEDesigns.jl for producing optimal "static experimental designs," i.e., +arrangements of experiments that exist along a Pareto-optimal tradeoff between cost and information gain. +We also show an example with synthetic data. + +## Setting + +Consider the following scenario. There exists a set of experiments, each of which, when performed, yields +measurements on one or more observables (features). Each subset of observables (and therefore each subset of experiments) +has some "information value," which is intentionally vaguely defined for generality, but for example, may be +a loss function if that subset is used to train some machine learning model. It is generally the value of acquiring that information. +Finally, each experiment has some monetary cost and execution time to perform the experiment, and +the user has some known tradeoff between overall execution time and cost. + +CEEDesigns.jl provides tools to take these inputs and produce a set of optimal "arrangements" of experiments for each +subset of experiments that form a Pareto front along the tradeoff between information gain and total combined cost +(monetary and time). This allows informed decisions to be made, for example, regarding how to allocate scarce +resources to a set of experiments that attain some acceptable level of information (or, conversely, reduce +uncertainty below some level). + +The arrangements produced by the tools introduced in this tutorial are called "static" because they inherently +assume that for each experiment, future data will deterministically yield the same information gain as the "historical" data did. +This information gain from the "historical" data is quantified based on certain aggregate statistics. + +We can also consider "generative experimental designs," where the information gain is modeled as a random variable. This concept is detailed in another [tutorial](./SimpleGenerative.jl). + +This tutorial introduces the theoretical framework behind static experimental designs with synthetic data. +For examples using real data, please see our other tutorials. + +## Theoretical Framework + +### Experiments + +Let $E = \{ e_1, \ldots, e_n\}$ be a set of $n$ experiments (i.e., $|E|=n$). Each experiment $e \in E$ has an +associated tuple $(m_{e},t_{e})$, giving the monetary cost and time duration required to perform experiment $e$. + +![experiments](assets/static_experiments.png) + +Consider $P(E)$, the power set of experiments (i.e., every possible subset of experiments). Each subset of +experiments $S\in P(E)$ has an associated value $v_{S}$, which is the value of the experiments contained in $S$. +This may be given by the loss function associated with a prediction task where the information yielded from $S$ +is used as predictor variables, or some other notion of information value. + +![experiments](assets/static_powerset.png) + +### Arrangements + +If experiments within a subset $S$ can be performed simultaneously (in parallel), then each $S$ may be arranged +optimally with respect to time. An arrangement $O_{S}$ of $S$ is a partition of the experiments in $S$ such that +the size of each partition is not larger than the maximum number of experiments that may be done in parallel. + +Let $l$ be the number of partitions, and $o_{i}$ a partition in $O_{S}$. Then the arrangement is a surjection from $S$ +onto $O_{S}$. If no experiments can be done in parallel, then $l=|S|$. If all experiments are done in parallel, then +$l=1$. Other arrangements fall between these extremes. + +![experiments](assets/static_arrangement.png) + +### Optimal Arrangements + +To find the optimal arrangement for each $S$ we need to know the cost of $O_{S}$. The monetary cost of $O_{S}$ is simply +the sum of the costs of each experiment: $m_{O_{S}}=\sum_{e\in S} m_{e}$. +The total time required is the sum of the maximum time *of each partition*. This is because while each partition in the +arrangement is done in serial, experiments within partitions are done in parallel. That is, $t_{O_{S}}=\sum_{i=1}^{l} \text{max} \{ t_{e}: e \in o_{i}\}$. +Given these costs and a parameter $\lambda$ which controls the tradeoff between monetary cost and time, the combined +cost of an arrangement is: $\lambda m_{O_{S}} + (1-\lambda) t_{O_{S}}$. + +For instance, consider the experiments $S = \{e_{1},e_{2},e_{3},e_{4}\}$, with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$. +If we conduct experiments $e_1$ through $e_4$ in sequence, this would correspond to an arrangement +$O_{S} = (\{ e_1 \}, \{ e_2 \}, \{ e_3 \}, \{ e_4 \})$ with a total cost of $m_{O_{S}} = 4$ and $t_{O_{S}} = 10$. + +However, if we decide to conduct $e_1$ in parallel with $e_3$, and $e_2$ with $e_4$, we would obtain an arrangement +$O_{S} = (\{ e_1, e_3 \}, \{ e_2, e_4 \})$ with a total cost of $m_{O_{S}} = 4$, and $t_{O_{S}} = 3 + 4 = 7$. + +Continuing our example and assuming a maximum of two parallel experiments, the optimal arrangement is to conduct +$e_1$ in parallel with $e_2$, and $e_3$ with $e_4$. This results in an arrangement $O_{S} = (\{ e_1, e_2 \}, \{ e_3, e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 2 + 4 = 6$. + +In fact, it can be readily demonstrated that the optimal arrangement can be found by ordering the experiments in +S in descending order according to their execution times. Consequently, the experiments are grouped sequentially +into partitions whose size equals the maximum number of parallel experiments, except possibly for the final set, +if the maximum number of parallel experiments does not divide $S$ evenly. + +## Synthetic Data Example + +We now present an example of finding cost-efficient designs for synthetic data using the CEEDesigns.jl package. + +First we load necessary packages. + +````@example SimpleStatic +using CEEDesigns, CEEDesigns.StaticDesigns +using Combinatorics: powerset +using DataFrames +using POMDPs, POMDPTools, MCTS +```` + +We consider a situation where there are 3 experiments, and we draw a value of their "loss function" +or "entropy" from the uniform distribution on the unit interval for each. + +For each $S\in P(E)$, we simulate the information value ($v_{S}$) of $S$ as the product of +the values for each individual experiment. +Therefore, because smaller values are better, any subset containing multiple experiments is guaranteed to be +more "valuable" than any component experiment. + +````@example SimpleStatic +experiments = ["e1", "e2", "e3"]; +experiments_val = Dict([e => rand() for e in experiments]); + +experiments_evals = Dict(map(Set.(collect(powerset(experiments)))) do s + if length(s) > 0 + s => prod([experiments_val[i] for i in s]) + else + return s => 1.0 + end +end); +nothing #hide +```` + +See our other tutorial on [heart disease triage](StaticDesigns.md) for an example of using CEEDesigns.jl's built-in +compatability with machine learning models from `MLJ.jl` to evalute performance of experiments using +predictive accuracy as information value. + +Next we set up the costs $(m_{e},t_{e})$ for each experiment. +Better experiments are more costly, both in terms of time and monetary cost. We print +the data frame showing each experiment and its associated costs. + +````@example SimpleStatic +experiments_costs = Dict( + sort(collect(keys(experiments_val)); by = k -> experiments_val[k], rev = true) .=> + tuple.(1:3, 1:3), +); + +DataFrame(; + experiment = collect(keys(experiments_costs)), + time = getindex.(values(experiments_costs), 1), + cost = getindex.(values(experiments_costs), 2), +) +```` + +We can plot the experiments ordered by their information value. + +````@example SimpleStatic +plot_evals( + experiments_evals; + f = x -> sort(collect(keys(x)); by = k -> x[k], rev = true), + ylabel = "loss", +) +```` + +We print the data frame showing each subset of experiments and its overall information value. + +````@example SimpleStatic +df_values = DataFrame(; + S = collect.(collect(keys(experiments_evals))), + value = collect(values(experiments_evals)), +) + +sort(df_values, order(:value; rev = true)) +```` + +Now we are ready to find the subsets of experiments giving an optimal tradeoff between information +value and combined cost. CEED exports a function `efficient_designs` +which formulates the problem of finding optimal arrangements as a Markov Decision Process and solves +optimal arrangements for each subset on the Pareto frontier. + +We set $\lambda=0.5$, the parameter controlling the relative weight given to monetary versus time costs +with the tuple `tradeoff`. + +````@example SimpleStatic +max_parallel = 2; +tradeoff = (0.5, 0.5); + +designs = efficient_designs( + experiments_costs, + experiments_evals; + max_parallel = max_parallel, + tradeoff = tradeoff, +); +nothing #hide +```` + +Finally we may produce a plot of the set of cost-efficient experimental designs. The set of designs +is plotted along a Pareto frontier giving tradeoff between informatio value (y-axis) and combined cost (x-axis). +Note that because we set the maximum number of parallel experiments equal to 2, the efficient design for the complete set +of experiments groups the experiments with long execution times together (see plot legend; each group within a partition is +prefixed with a number). + +````@example SimpleStatic +plot_front(designs; labels = make_labels(designs), ylabel = "loss") +```` + diff --git a/docs/src/tutorials/StaticDesigns.jl b/docs/src/tutorials/StaticDesigns.jl index 2a48a45..022cf0c 100644 --- a/docs/src/tutorials/StaticDesigns.jl +++ b/docs/src/tutorials/StaticDesigns.jl @@ -1,38 +1,20 @@ # # Heart Disease Triage # Consider a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. - -# ## Theoretical Framework - -# Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$. - -# For each subset $S \subseteq E$ of experiments, we denote by $v_S$ the value of information acquired from conducting experiments in $S$. - -# In the cost-sensitive setting of CEEDesigns, conducting an experiment $e$ incurs a cost $(m_e, t_e)$. Generally, this cost is specified in terms of monetary cost and execution time of the experiment. - -# To compute the cost associated with carrying out a set of experiments $S$, we first need to introduce the notion of an arrangement $o$ of the experiments $S$. An arrangement is modeled as a sequence of mutually disjoint subsets of $S$. In other words, $o = (o_1, \ldots, o_l)$ for a given $l\in\mathbb N$, where $\bigcup_{i=1}^l o_i = S$ and $o_i \cap o_j = \emptyset$ for each $1\leq i < j \leq l$. - -# Given a subset $S$ of experiments and their arrangement $o$, the total monetary cost and execution time of the experimental design is given as $m_o = \sum_{e\in S} m_e$ and $t_o = \sum_{i=1}^l \max \{ t_e : e\in o_i\}$, respectively. - -# For instance, consider the experiments $e_1,\, e_2,\, e_3$, and $e_4$ with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$. If we conduct experiments $e_1$ through $e_4$ in sequence, this would correspond to an arrangement $o = (\{ e_1 \}, \{ e_2 \}, \{ e_3 \}, \{ e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 10$. - -# However, if we decide to conduct $e_1$ in parallel with $e_3$, and $e_2$ with $e_4$, we would obtain an arrangement $o = (\{ e_1, e_3 \}, \{ e_2, e_4 \})$ with a total cost of $m_o = 4$, and $t_o = 3 + 4 = 7$. - -# Given the constraint on the maximum number of parallel experiments, we devise an arrangement $o$ of experiments $S$ such that, for a fixed tradeoff between monetary cost and execution time, the expected combined cost $c_{(o, \lambda)} = \lambda m_o + (1-\lambda) t_o$ is minimized (i.e., the execution time is minimized). - -# In fact, it can be readily demonstrated that the optimal arrangement can be found by ordering the experiments in set $S$ in descending order according to their execution times. Consequently, the experiments are grouped sequentially into sets whose size equals the maximum number of parallel experiments, except possibly for the final set. - -# Continuing our example and assuming a maximum of two parallel experiments, the optimal arrangement is to conduct $e_1$ in parallel with $e_2$, and $e_3$ with $e_4$. This results in an arrangement $o = (\{ e_1, e_2 \}, \{ e_3, e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 2 + 4 = 6$. - -# Assuming the information values $v_S$ and optimized experimental costs $c_S$ for each subset $S \subseteq E$ of experiments, we then generate a set of cost-efficient experimental designs. +# For details on the theoretical background and notation, please see our tutorial on [static experimental designs](SimpleStatic.md), this tutorial +# is a concrete application of the tools described in that document. # ### Application to Predictive Modeling +# Let's generalize the example from [static experimental designs](SimpleStatic.md) to the case where we want to compute the information value $v_{S}$ as the predictive +# ability of a machine learning model which uses the measurements gained from experiments in $S$ to predict some $y$ of interest. +# +# Let's introduce some formal notation. # Consider a dataset of historical readouts over $m$ features $X = \{x_1, \ldots, x_m\}$, and let $y$ denote the target variable that we want to predict. - -# We assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. +# Assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. # Then, for each subset $S \subseteq E$ of experiments, we may model the value of information acquired by conducting the experiments in $S$ as the accuracy of a predictive model that predicts the value of $y$ based on readouts over features in $X_S = \bigcup_{e\in S} X_e$. +# Then this accuracy is our information value $v_{S}$ of $S$. # ## Heart Disease Dataset @@ -44,7 +26,7 @@ data[1:10, :] # ## Predictive Accuracy -# The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +# The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. # We specify the experiments along with the associated features: @@ -96,7 +78,7 @@ model = classifier(; n_trees = 20, max_depth = 10) # ### Performance Evaluation -# We use `evaluate_experiments` from `CEEDesigns.StaticDesigns` to evaluate the predictive accuracy over subsets of experiments. We use `LogLoss` as a measure of accuracy. It is possible to pass additional keyword arguments, which will be passed to `MLJ.evaluate` (such as `measure`, shown below). +# We use `evaluate_experiments` from `CEEDesigns.StaticDesigns` to evaluate the predictive accuracy over subsets of experiments. We use `LogLoss` as a measure of accuracy. It is possible to provide additional keyword arguments, which will be passed to `MLJ.evaluate` (such as `measure`, shown below). using CEEDesigns, CEEDesigns.StaticDesigns @@ -112,8 +94,12 @@ perf_eval = evaluate_experiments( measure = LogLoss(), ) -# We plot performance measures evaluated for subsets of experiments. -plot_evals(perf_eval; ylabel = "logloss") +# We plot performance measures evaluated for subsets of experiments, sorted by performance measure. +plot_evals( + perf_eval; + f = x -> sort(collect(keys(x)); by = k -> x[k], rev = true), + ylabel = "logloss", +) # ## Cost-Efficient Designs @@ -134,7 +120,7 @@ plot_front(designs; labels = make_labels(designs), ylabel = "logloss") # ### Parallel Experiments -# We may exploit parallelism in the experimental arrangement. To that end, we first specify the monetary cost and execution time for each experiment, respectively. +# The previous example assumed that experiments had to be run sequentially. We can see how the optimal arrangement changes if we assume multiple experiments can be run in parallel. To that end, we first specify the monetary cost and execution time for each experiment, respectively. experiments_costs = Dict( ## experiment => (monetary cost, execution time) => features @@ -144,7 +130,7 @@ experiments_costs = Dict( "BloodSugar" => (20.0, 20.0) => ["FastingBS"], ) -# We set the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. +# We set the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time. In our case, we aim to minimize the execution time. # Below, we demonstrate the flexibility of `efficient_designs` as it can both evaluate the performance of experiments and generate efficient designs. Internally, `evaluate_experiments` is called first, followed by `efficient_designs`. Keyword arguments to the respective functions has to wrapped in `eval_options` and `arrangement_options` named tuples, respectively. diff --git a/docs/src/tutorials/StaticDesigns.md b/docs/src/tutorials/StaticDesigns.md index 49685eb..0ef41bb 100644 --- a/docs/src/tutorials/StaticDesigns.md +++ b/docs/src/tutorials/StaticDesigns.md @@ -5,38 +5,20 @@ EditURL = "StaticDesigns.jl" # Heart Disease Triage Consider a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. - -## Theoretical Framework - -Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$. - -For each subset $S \subseteq E$ of experiments, we denote by $v_S$ the value of information acquired from conducting experiments in $S$. - -In the cost-sensitive setting of CEEDesigns, conducting an experiment $e$ incurs a cost $(m_e, t_e)$. Generally, this cost is specified in terms of monetary cost and execution time of the experiment. - -To compute the cost associated with carrying out a set of experiments $S$, we first need to introduce the notion of an arrangement $o$ of the experiments $S$. An arrangement is modeled as a sequence of mutually disjoint subsets of $S$. In other words, $o = (o_1, \ldots, o_l)$ for a given $l\in\mathbb N$, where $\bigcup_{i=1}^l o_i = S$ and $o_i \cap o_j = \emptyset$ for each $1\leq i < j \leq l$. - -Given a subset $S$ of experiments and their arrangement $o$, the total monetary cost and execution time of the experimental design is given as $m_o = \sum_{e\in S} m_e$ and $t_o = \sum_{i=1}^l \max \{ t_e : e\in o_i\}$, respectively. - -For instance, consider the experiments $e_1,\, e_2,\, e_3$, and $e_4$ with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$. If we conduct experiments $e_1$ through $e_4$ in sequence, this would correspond to an arrangement $o = (\{ e_1 \}, \{ e_2 \}, \{ e_3 \}, \{ e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 10$. - -However, if we decide to conduct $e_1$ in parallel with $e_3$, and $e_2$ with $e_4$, we would obtain an arrangement $o = (\{ e_1, e_3 \}, \{ e_2, e_4 \})$ with a total cost of $m_o = 4$, and $t_o = 3 + 4 = 7$. - -Given the constraint on the maximum number of parallel experiments, we devise an arrangement $o$ of experiments $S$ such that, for a fixed tradeoff between monetary cost and execution time, the expected combined cost $c_{(o, \lambda)} = \lambda m_o + (1-\lambda) t_o$ is minimized (i.e., the execution time is minimized). - -In fact, it can be readily demonstrated that the optimal arrangement can be found by ordering the experiments in set $S$ in descending order according to their execution times. Consequently, the experiments are grouped sequentially into sets whose size equals the maximum number of parallel experiments, except possibly for the final set. - -Continuing our example and assuming a maximum of two parallel experiments, the optimal arrangement is to conduct $e_1$ in parallel with $e_2$, and $e_3$ with $e_4$. This results in an arrangement $o = (\{ e_1, e_2 \}, \{ e_3, e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 2 + 4 = 6$. - -Assuming the information values $v_S$ and optimized experimental costs $c_S$ for each subset $S \subseteq E$ of experiments, we then generate a set of cost-efficient experimental designs. +For details on the theoretical background and notation, please see our tutorial on [static experimental designs](SimpleStatic.md), this tutorial +is a concrete application of the tools described in that document. ### Application to Predictive Modeling -Consider a dataset of historical readouts over $m$ features $X = \{x_1, \ldots, x_m\}$, and let $y$ denote the target variable that we want to predict. +Let's generalize the example from [static experimental designs](SimpleStatic.md) to the case where we want to compute the information value $v_{S}$ as the predictive +ability of a machine learning model which uses the measurements gained from experiments in $S$ to predict some $y$ of interest. -We assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. +Let's introduce some formal notation. +Consider a dataset of historical readouts over $m$ features $X = \{x_1, \ldots, x_m\}$, and let $y$ denote the target variable that we want to predict. +Assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. Then, for each subset $S \subseteq E$ of experiments, we may model the value of information acquired by conducting the experiments in $S$ as the accuracy of a predictive model that predicts the value of $y$ based on readouts over features in $X_S = \bigcup_{e\in S} X_e$. +Then this accuracy is our information value $v_{S}$ of $S$. ## Heart Disease Dataset @@ -50,7 +32,7 @@ data[1:10, :] ## Predictive Accuracy -The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. We specify the experiments along with the associated features: @@ -120,7 +102,7 @@ model = classifier(; n_trees = 20, max_depth = 10) ### Performance Evaluation -We use `evaluate_experiments` from `CEEDesigns.StaticDesigns` to evaluate the predictive accuracy over subsets of experiments. We use `LogLoss` as a measure of accuracy. It is possible to pass additional keyword arguments, which will be passed to `MLJ.evaluate` (such as `measure`, shown below). +We use `evaluate_experiments` from `CEEDesigns.StaticDesigns` to evaluate the predictive accuracy over subsets of experiments. We use `LogLoss` as a measure of accuracy. It is possible to provide additional keyword arguments, which will be passed to `MLJ.evaluate` (such as `measure`, shown below). ````@example StaticDesigns using CEEDesigns, CEEDesigns.StaticDesigns @@ -138,10 +120,14 @@ perf_eval = evaluate_experiments( ) ```` -We plot performance measures evaluated for subsets of experiments. +We plot performance measures evaluated for subsets of experiments, sorted by performance measure. ````@example StaticDesigns -plot_evals(perf_eval; ylabel = "logloss") +plot_evals( + perf_eval; + f = x -> sort(collect(keys(x)); by = k -> x[k], rev = true), + ylabel = "logloss", +) ```` ## Cost-Efficient Designs @@ -170,7 +156,7 @@ plot_front(designs; labels = make_labels(designs), ylabel = "logloss") ### Parallel Experiments -We may exploit parallelism in the experimental arrangement. To that end, we first specify the monetary cost and execution time for each experiment, respectively. +The previous example assumed that experiments had to be run sequentially. We can see how the optimal arrangement changes if we assume multiple experiments can be run in parallel. To that end, we first specify the monetary cost and execution time for each experiment, respectively. ````@example StaticDesigns experiments_costs = Dict( @@ -182,7 +168,7 @@ experiments_costs = Dict( ) ```` -We set the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. +We set the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time. In our case, we aim to minimize the execution time. Below, we demonstrate the flexibility of `efficient_designs` as it can both evaluate the performance of experiments and generate efficient designs. Internally, `evaluate_experiments` is called first, followed by `efficient_designs`. Keyword arguments to the respective functions has to wrapped in `eval_options` and `arrangement_options` named tuples, respectively. diff --git a/docs/src/tutorials/StaticDesignsFiltration.jl b/docs/src/tutorials/StaticDesignsFiltration.jl index 258841e..0b57bf2 100644 --- a/docs/src/tutorials/StaticDesignsFiltration.jl +++ b/docs/src/tutorials/StaticDesignsFiltration.jl @@ -1,26 +1,27 @@ # # Heart Disease Triage With Early Droupout -# Consider again a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. +# Consider again a situation where a group of patients is tested for a specific disease. +# It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. # Moreover, we may assume that for some patients, the evidence gathered from proxy experiments can be considered 'conclusive'. Effectively, some patients may not need any additional triage; they might be deemed healthy or require immediate commencement of treatment. By doing so, we could save significant resources. # ## Theoretical Framework -# Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$. +# We take as a basis the setup and notation from the basic framework presented in the [static experimental designs tutorial](SimpleStatic.md). -# For each subset $S \subseteq E$ of experiments, we denote by $v_S$ the value of information acquired from conducting experiments in $S$. - -# Moreover, it can be assumed that a set of extrinsic decision-making rules is imposed on the experimental readouts. If the experimental evidence acquired for a given entity satisfies a specific criterion, that entity is then removed from the triage. However, other entities within the batch will generally continue in the experimental process. In general, the process of establishing such rules is largely dependent on the specific problem and requires comprehensive understanding of the subject area. +# We again have a set of experiments $E$, but now assume that a set of extrinsic decision-making rules is imposed on the experimental readouts. +# If the experimental evidence acquired for a given entity satisfies a specific criterion, that entity is then removed from the triage. +# However, other entities within the batch will generally continue in the experimental process. +# In general, the process of establishing such rules is largely dependent on the specific problem and requires comprehensive understanding of the subject area. # We denote the expected fraction of entities that remain in the triage after conducting a set $S$ of experiments as the filtration rate, $f_S$. In the context of disease triage, this can be interpreted as the fraction of patients for whom the experimental evidence does not provide a 'conclusive' result. -# In the cost-sensitive setting of CEEDesigns, conducting an experiment $e$ incurs a cost $(m_e, t_e)$. Generally, this cost is specified in terms of monetary cost and execution time of the experiment. - -# To compute the cost associated with carrying out a set of experiments $S$, we first need to introduce the notion of an arrangement $o$ of the experiments $S$. An arrangement is modeled as a sequence of mutually disjoint subsets of $S$. In other words, $o = (o_1, \ldots, o_l)$ for a given $l\in\mathbb N$, where $\bigcup_{i=1}^l o_i = S$ and $o_i \cap o_j = \emptyset$ for each $1\leq i < j \leq l$. +# As previously, each experiment $e$ incurs a cost $(m_e, t_e)$. Again, we let $O_{S}$ denote an arrangement of experiments in $S$. -# Given a subset $S$ of experiments and their arrangement $o$, the total (discounted) monetary cost and execution time of the experimental design is given as $m_o = \sum{i=1}^l r_{S_{i-1}}\sum_{e\in o_i} m_e$ and $t_o = \sum_{i=1}^l \max \{ t_e : e\in o_i\}$, respectively. Importantly, the factor $r_{S_{i-1}}$ models the fact that a set of entities may have dropped out in the previous experiments, hence saving the resources on running the subsequent experiments. +# Given a subset $S$ of experiments and their arrangement $O_{S}$, the total (discounted) monetary cost and execution time of the experimental design is given as $m_o = \sum_{i=1}^{l} r_{S_{i-1}}\sum_{e\in o_i} m_e$ and $t_o = \sum_{i=1}^{l} \max \{ t_e : e\in o_i\}$, respectively. +# Importantly, the new factor $r_{S_{i-1}}$ models the fact that a set of entities may have dropped out in the previous experiments, hence saving the resources on running the subsequent experiments. -# We note that these computations are based on the assumption that monetary cost is associated with the analysis of a single experimental entity, such as a patient. Therefore, the total monetary cost obtained for a specific arrangement is effectively the ['expected'](https://en.wikipedia.org/wiki/Expected_value) monetary cost, adjusted for a single entity. Conversely, we suppose that all entities can be concurrently examined in a specific experiment. As such, the total execution time is equivalent to the longest time until all experiments within an arrangement are finished or all entities have been eliminated (which ocurrs when the filtration rate the experiments conducted so far is zero). Importantly, this distinctly differs from calculating the 'expected lifespan' of an entity in the triage. +# We note that these computations are based on the assumption that monetary cost is associated with the analysis of a single experimental entity, such as a patient. Therefore, the total monetary cost obtained for a specific arrangement is effectively the ["expected"](https://en.wikipedia.org/wiki/Expected_value) monetary cost, adjusted for a single entity. Conversely, we suppose that all entities can be concurrently examined in a specific experiment. As such, the total execution time is equivalent to the longest time until all experiments within an arrangement are finished or all entities have been eliminated (which ocurrs when the filtration rate the experiments conducted so far is zero). Importantly, this distinctly differs from calculating the 'expected lifespan' of an entity in the triage. # For instance, consider the experiments $e_1,\, e_2,\, e_3$, and $e_4$ with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$, and filtration rates $0.9,\,0.8,\,0.7$, and $0.6$. For subsets of experiments, we simply assume that the filtration rates multiply, thereby treating the experiments as independent binary discriminators. In other words, the filtration rate for a set $S=\{ e_1, e_3 \}$ would equal $f_S = 0.9 * 0.7 = 0.63$. @@ -42,7 +43,7 @@ # - _actions_ are subsets of experiments which have not been conducted yet; the size of these subsets is restricted by the maximum number of parallel experiments; # - _reward_ is a combined monetary cost and execution time, discounted by the filtration rate of previously conducted experiments. -# Provided we know the information values $v_S$, filtration rates $r_S$, and experimental costs $c_S$ for each subset $S \subseteq E$, we find a collection of Pareto-efficient experimental designs that balance both the implied value of information and the experimental cost. +# Provided we know the information values $v_S$, filtration rates $r_S$, and experimental costs $c_S$ for each set of experiments $S$, we find a collection of Pareto-efficient experimental designs that balance both the implied value of information and the experimental cost. # ## Heart Disease Dataset @@ -90,7 +91,7 @@ data_binary[1:10, :] # In this scenario, we model the value of information $v_S$ acquired by conducting a set of experiments as the ratio of patients for whom the results across the experiments in $S$ were 'inconclusive', i.e., $|\cap_{e\in S}\{ \text{patient} : \text{inconclusive in } e \}| / |\text{patients}|$. Essentially, the very same measure is used here to estimate the filtration rate. -# The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +# The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. # We specify the experiments along with the associated features: diff --git a/docs/src/tutorials/StaticDesignsFiltration.md b/docs/src/tutorials/StaticDesignsFiltration.md index 622a139..97e2264 100644 --- a/docs/src/tutorials/StaticDesignsFiltration.md +++ b/docs/src/tutorials/StaticDesignsFiltration.md @@ -4,27 +4,28 @@ EditURL = "StaticDesignsFiltration.jl" # Heart Disease Triage With Early Droupout -Consider again a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. +Consider again a situation where a group of patients is tested for a specific disease. +It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. Moreover, we may assume that for some patients, the evidence gathered from proxy experiments can be considered 'conclusive'. Effectively, some patients may not need any additional triage; they might be deemed healthy or require immediate commencement of treatment. By doing so, we could save significant resources. ## Theoretical Framework -Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$. +We take as a basis the setup and notation from the basic framework presented in the [static experimental designs tutorial](SimpleStatic.md). -For each subset $S \subseteq E$ of experiments, we denote by $v_S$ the value of information acquired from conducting experiments in $S$. - -Moreover, it can be assumed that a set of extrinsic decision-making rules is imposed on the experimental readouts. If the experimental evidence acquired for a given entity satisfies a specific criterion, that entity is then removed from the triage. However, other entities within the batch will generally continue in the experimental process. In general, the process of establishing such rules is largely dependent on the specific problem and requires comprehensive understanding of the subject area. +We again have a set of experiments $E$, but now assume that a set of extrinsic decision-making rules is imposed on the experimental readouts. +If the experimental evidence acquired for a given entity satisfies a specific criterion, that entity is then removed from the triage. +However, other entities within the batch will generally continue in the experimental process. +In general, the process of establishing such rules is largely dependent on the specific problem and requires comprehensive understanding of the subject area. We denote the expected fraction of entities that remain in the triage after conducting a set $S$ of experiments as the filtration rate, $f_S$. In the context of disease triage, this can be interpreted as the fraction of patients for whom the experimental evidence does not provide a 'conclusive' result. -In the cost-sensitive setting of CEEDesigns, conducting an experiment $e$ incurs a cost $(m_e, t_e)$. Generally, this cost is specified in terms of monetary cost and execution time of the experiment. - -To compute the cost associated with carrying out a set of experiments $S$, we first need to introduce the notion of an arrangement $o$ of the experiments $S$. An arrangement is modeled as a sequence of mutually disjoint subsets of $S$. In other words, $o = (o_1, \ldots, o_l)$ for a given $l\in\mathbb N$, where $\bigcup_{i=1}^l o_i = S$ and $o_i \cap o_j = \emptyset$ for each $1\leq i < j \leq l$. +As previously, each experiment $e$ incurs a cost $(m_e, t_e)$. Again, we let $O_{S}$ denote an arrangement of experiments in $S$. -Given a subset $S$ of experiments and their arrangement $o$, the total (discounted) monetary cost and execution time of the experimental design is given as $m_o = \sum{i=1}^l r_{S_{i-1}}\sum_{e\in o_i} m_e$ and $t_o = \sum_{i=1}^l \max \{ t_e : e\in o_i\}$, respectively. Importantly, the factor $r_{S_{i-1}}$ models the fact that a set of entities may have dropped out in the previous experiments, hence saving the resources on running the subsequent experiments. +Given a subset $S$ of experiments and their arrangement $O_{S}$, the total (discounted) monetary cost and execution time of the experimental design is given as $m_o = \sum_{i=1}^{l} r_{S_{i-1}}\sum_{e\in o_i} m_e$ and $t_o = \sum_{i=1}^{l} \max \{ t_e : e\in o_i\}$, respectively. +Importantly, the new factor $r_{S_{i-1}}$ models the fact that a set of entities may have dropped out in the previous experiments, hence saving the resources on running the subsequent experiments. -We note that these computations are based on the assumption that monetary cost is associated with the analysis of a single experimental entity, such as a patient. Therefore, the total monetary cost obtained for a specific arrangement is effectively the ['expected'](https://en.wikipedia.org/wiki/Expected_value) monetary cost, adjusted for a single entity. Conversely, we suppose that all entities can be concurrently examined in a specific experiment. As such, the total execution time is equivalent to the longest time until all experiments within an arrangement are finished or all entities have been eliminated (which ocurrs when the filtration rate the experiments conducted so far is zero). Importantly, this distinctly differs from calculating the 'expected lifespan' of an entity in the triage. +We note that these computations are based on the assumption that monetary cost is associated with the analysis of a single experimental entity, such as a patient. Therefore, the total monetary cost obtained for a specific arrangement is effectively the ["expected"](https://en.wikipedia.org/wiki/Expected_value) monetary cost, adjusted for a single entity. Conversely, we suppose that all entities can be concurrently examined in a specific experiment. As such, the total execution time is equivalent to the longest time until all experiments within an arrangement are finished or all entities have been eliminated (which ocurrs when the filtration rate the experiments conducted so far is zero). Importantly, this distinctly differs from calculating the 'expected lifespan' of an entity in the triage. For instance, consider the experiments $e_1,\, e_2,\, e_3$, and $e_4$ with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$, and filtration rates $0.9,\,0.8,\,0.7$, and $0.6$. For subsets of experiments, we simply assume that the filtration rates multiply, thereby treating the experiments as independent binary discriminators. In other words, the filtration rate for a set $S=\{ e_1, e_3 \}$ would equal $f_S = 0.9 * 0.7 = 0.63$. @@ -46,7 +47,7 @@ When we allow simultaneous execution of experiments, the problem turns more comp - _actions_ are subsets of experiments which have not been conducted yet; the size of these subsets is restricted by the maximum number of parallel experiments; - _reward_ is a combined monetary cost and execution time, discounted by the filtration rate of previously conducted experiments. -Provided we know the information values $v_S$, filtration rates $r_S$, and experimental costs $c_S$ for each subset $S \subseteq E$, we find a collection of Pareto-efficient experimental designs that balance both the implied value of information and the experimental cost. +Provided we know the information values $v_S$, filtration rates $r_S$, and experimental costs $c_S$ for each set of experiments $S$, we find a collection of Pareto-efficient experimental designs that balance both the implied value of information and the experimental cost. ## Heart Disease Dataset @@ -100,7 +101,7 @@ data_binary[1:10, :] In this scenario, we model the value of information $v_S$ acquired by conducting a set of experiments as the ratio of patients for whom the results across the experiments in $S$ were 'inconclusive', i.e., $|\cap_{e\in S}\{ \text{patient} : \text{inconclusive in } e \}| / |\text{patients}|$. Essentially, the very same measure is used here to estimate the filtration rate. -The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. We specify the experiments along with the associated features: diff --git a/docs/src/tutorials/assets/generative_actions.png b/docs/src/tutorials/assets/generative_actions.png new file mode 100644 index 0000000..11b779c Binary files /dev/null and b/docs/src/tutorials/assets/generative_actions.png differ diff --git a/docs/src/tutorials/assets/generative_historical.png b/docs/src/tutorials/assets/generative_historical.png new file mode 100644 index 0000000..eadbb6e Binary files /dev/null and b/docs/src/tutorials/assets/generative_historical.png differ diff --git a/docs/src/tutorials/assets/generative_search.png b/docs/src/tutorials/assets/generative_search.png new file mode 100644 index 0000000..a99683f Binary files /dev/null and b/docs/src/tutorials/assets/generative_search.png differ diff --git a/docs/src/tutorials/assets/generative_state.png b/docs/src/tutorials/assets/generative_state.png new file mode 100644 index 0000000..5e9dfeb Binary files /dev/null and b/docs/src/tutorials/assets/generative_state.png differ diff --git a/docs/src/tutorials/assets/generative_weights.png b/docs/src/tutorials/assets/generative_weights.png new file mode 100644 index 0000000..7f9c21f Binary files /dev/null and b/docs/src/tutorials/assets/generative_weights.png differ diff --git a/docs/src/tutorials/assets/static_arrangement.png b/docs/src/tutorials/assets/static_arrangement.png new file mode 100644 index 0000000..d9a954f Binary files /dev/null and b/docs/src/tutorials/assets/static_arrangement.png differ diff --git a/docs/src/tutorials/assets/static_experiments.png b/docs/src/tutorials/assets/static_experiments.png new file mode 100644 index 0000000..9f63d7a Binary files /dev/null and b/docs/src/tutorials/assets/static_experiments.png differ diff --git a/docs/src/tutorials/assets/static_powerset.png b/docs/src/tutorials/assets/static_powerset.png new file mode 100644 index 0000000..d514bcd Binary files /dev/null and b/docs/src/tutorials/assets/static_powerset.png differ diff --git a/docs/src/tutorials/data/class.csv b/docs/src/tutorials/data/class.csv new file mode 100644 index 0000000..dda6e44 --- /dev/null +++ b/docs/src/tutorials/data/class.csv @@ -0,0 +1,1001 @@ +x1,x2,x3,x4,x5,y +-0.39988512043361646,1.3337073852278094,1.1399557169899495,1.1656889925296003,0.1028643166223937,0 +0.11604658695262537,0.12370814601071312,1.314537694005196,1.347981083362914,-1.4885043346989004,0 +0.17817098696507017,-0.4940225050295366,0.7101473442842716,0.6751970915184036,-1.4634551594417935,0 +-1.1712824011295664,0.04258831266945029,1.62025299893532,1.71087271300213,-1.449551858671816,0 +0.9131479655930531,-1.457986249717023,0.8826851445584001,1.1959448704064282,-2.504196488731152,0 +-0.3471865322162061,2.6476894973159713,1.1821266682914224,-0.37015731553094144,-0.387102983473691,1 +0.8146480451580892,0.5407223388443838,1.9284968461160732,1.9477565227431675,-2.072595692705426,0 +-1.863027358704509,-2.064648810239415,-1.2006661227151492,0.36567595689210564,1.7511483753093189,1 +-2.188227128112658,1.5366361127776336,3.5424988094574195,4.394714086659788,-1.1269768750094369,1 +0.25251658133550636,0.3031942523561524,0.9371226649945322,1.5767164463013135,-0.224483264186758,1 +0.002839304263283937,2.6693047445091818,0.6739775688676186,0.38393563984935275,1.5297046259184675,1 +-0.15765068614618127,-0.584960313389373,1.6174417357967157,1.7897073987828693,-2.3252752016010594,0 +-1.588671276196846,-0.7253429104252509,0.12871718758431017,1.6980802479231558,1.3763782270136524,1 +-0.5539795119187028,0.8641465276170883,1.1831377666655136,1.203119843349317,-0.3753730029975938,0 +-3.019655172797125,-2.353284160287712,-0.756619475585828,1.9363809339481173,2.5465320170459496,1 +-2.2302994428484015,3.698457373872727,1.702969084719038,-0.9199008126963157,-0.5191667065979122,0 +1.4219932437113503,-1.3852574357495575,0.15006304416893193,1.192437738256401,-0.9015312471067121,1 +0.33528525842650936,-1.0583267029426657,1.2005676128416884,3.8794998410712656,0.30601064761181784,1 +-0.99460690730578,1.4563728592217473,0.9150874112802068,0.4570560762560669,0.1680623705018709,0 +0.23140506706972497,1.551542933942709,-0.30803453985981544,-1.3114388637286236,0.7500362837752963,0 +0.09571378722300405,0.8766197248160615,0.5570712567023978,0.7246971078686033,0.34876599548223997,0 +-1.3126003249409937,0.38281298783407613,1.7629191942417348,1.686551435699979,-1.4197734111591906,0 +0.44218935006583726,0.8412828961933221,1.5222756065494971,1.420435629528647,-1.2820778759678264,0 +-0.5326566627094349,-0.9604334112597348,-0.8816563838958104,-0.15276630250099577,1.0987107055733827,1 +0.7302680218871171,0.9906695152209604,2.8332698250879425,2.9688234397952176,-2.5736099471570455,0 +-1.0392284305363786,0.7760894532153282,1.9320306308421653,2.0555619218077528,-1.1042882754484817,0 +-1.9026497433532115,-2.740984092566441,-1.8582830859721775,0.1439942741919047,2.37217632728401,1 +-1.7484369756318965,0.6450491389743567,1.8716757632544052,2.888407118614435,0.05982366085196411,1 +1.3590038150097192,0.3841468074075277,0.32990463800105696,-0.025743076177584312,-0.866611506814594,0 +-1.6479385244824136,1.97615375796191,0.942025099668057,0.5112315792164399,0.9052476952381469,1 +-0.5659567834195822,0.5410611803100303,1.5752955530716046,2.1174936224627854,-0.600558330585039,1 +-0.20189033568009584,0.10603824702787712,-1.892749279969117,-1.724799677479691,2.6777373807625984,1 +0.5845804037322986,-0.06893575100251259,0.8299095969712593,1.8216223685944646,-0.1897873243294168,1 +-2.3161910011712736,3.3421119396836843,2.1972670146871827,0.40907748426327806,-0.5339294539843672,0 +0.8104938049569008,-0.8774252601102237,-0.3540597257029062,0.03218458727704887,-0.2923174684756149,1 +0.9794765452898768,-1.3523156943512733,0.5737997850946803,1.92905005411041,-0.894629770316831,0 +-1.342998410449443,-2.073374717926688,0.03524748391112986,3.157990673440001,1.7678401006007478,1 +1.1454017623671824,0.15623894759253365,-0.47106695228735274,-0.07096741606110513,0.7900960055960877,1 +1.1230061111887255,1.8758659685891437,1.1688942064882903,0.8518335796669269,-0.2774673385828619,0 +-0.47399281769159063,0.3921123029107083,1.0336859393929647,0.5074139337344216,-1.2938429180683717,0 +0.39695144216302447,2.24500987841873,1.8358856567485207,1.9086361339525242,-0.051723598267559234,1 +-1.252657098940631,-2.2371993264915115,-0.8809433452083446,0.7864673676346161,1.0930513353458666,1 +1.227777218867132,0.5678791678041634,-1.3147754582220097,-1.1044983578133225,1.9996654624928685,1 +2.7177144448080233,3.279216220230244,1.0107162303305897,0.32047860051265065,0.3771511240585472,0 +0.34170742335316306,0.6574516825778183,1.5767034195334615,1.6443118646895647,-1.312720932913606,0 +-1.0273458541165177,3.139604040181855,0.5284307676359961,-1.3334780415299767,0.7940112516017646,0 +0.5000972392852311,1.2600194132774851,3.0589695375657433,3.1926302672835916,-2.504985636583183,0 +-1.5768471501965986,-3.101460742738223,-0.7210631890090313,2.644400635078993,2.0095535904774593,1 +-0.0687091133622284,1.1099363576465653,1.2278473373223706,1.5303631617872808,-0.03540755496737957,1 +1.694041836780091,1.3887269463273704,0.9545909077940009,0.5762107052063541,-0.7623208699884604,0 +0.17565053324161384,-1.8795306049358018,-0.7912307538040068,0.9100880580025064,0.8998991297086507,1 +0.974788666041509,-0.5259535966636866,-0.13528107648643845,-0.1621088101627579,-0.7180233235496399,1 +-1.2799898121065132,2.069540357367523,1.224501416319961,-0.02543480438022061,-0.3726961042499115,0 +-0.8160826168926728,-1.0682308984810533,-0.2599112999353876,0.5045564713834325,0.3632199808768607,1 +-1.966611004874168,5.143180093091348,1.5311724641757343,-1.368784808611951,0.7459099261638829,0 +0.7084421694524579,2.37660565267916,0.6903810953296443,0.13660478370532292,0.6890783285652184,1 +-1.042375598601283,1.0710852812053886,-0.3926509757574178,0.14210418355659382,2.4932493913690426,1 +1.2083511796741444,-1.6002672800658504,-0.9273725883632992,0.060702624189509,0.21475291526726004,1 +-0.5349426563093,-1.919233366573267,0.7862269265107295,1.271114692577681,-2.1736276053700023,0 +-2.0855064200523437,-1.286326457429396,-0.4501376037130722,1.7089853679249796,2.3393138066685957,0 +0.023181263887859016,0.26710729989978255,-0.24918411387759687,0.07418818332866017,0.9213559584808393,1 +-0.11262807820639742,1.9282367376968323,0.12211674809090606,-0.04524717905409026,1.6367544617469227,1 +-1.562266414044001,3.91140520095054,2.259827841834646,1.4046779660119169,0.7346946021136926,1 +-4.428295465939227,1.4790391846360043,1.6009527495553155,2.3948416743938106,1.8810237029346508,1 +-1.2299188052232597,-3.168167515709092,-1.682860328407192,0.3018860641404201,1.4840862898811502,1 +0.6406256766743919,1.1992443480007198,1.8213617472705823,2.2202243097677634,-0.8048284300813728,1 +-0.5524768731809919,-1.3367373265020586,1.47945027633624,2.362863854966524,-1.9942264502089997,1 +-1.1001449704883777,-0.09100743297252478,1.2519731055094478,0.4646891869028046,-2.122943291445078,0 +3.02340362402904,-1.7265826012417689,1.915617047167827,5.518951575192933,-1.123575426541911,1 +0.5495796428297566,-2.7117422459451483,1.1315243392902423,3.2200323394406536,-1.9888898082974436,0 +0.6206020949277804,-0.021078667227930614,1.3312176941033347,1.4782771594850999,-1.6982065854048138,0 +-0.6383069610907426,-1.0231723798559047,-0.022244641924678565,1.1430843745850978,0.49903726143328453,1 +-1.602436550541821,2.2815144150032376,1.6988488486678965,-0.28180691224058774,-1.4378588704251127,0 +2.543880393273679,2.2636916545767933,2.254406352185062,1.827698094768528,-1.8159189566568443,0 +-0.9456358190090774,1.0727250490293492,1.089117177205299,1.7046145517840432,0.7360756700932674,1 +-2.9811845626549314,-0.3687736065870666,-0.25115782032436185,0.8121624506753856,2.1090210976833563,1 +-2.440418840792269,-2.3753997793684407,-1.5770626655484015,0.4162607926879637,2.564656269219452,1 +-0.966778564636996,-1.851613260642643,-1.0067051261601625,0.13061384287983646,0.9539798946779102,1 +-0.3310629620072062,0.9215548687347038,1.2111534754172626,1.0850436334684088,-0.5878063620298832,0 +-1.2999968705058151,0.8088230896893808,1.6297627085941842,1.0429458990187492,-1.3959641305232235,0 +0.8394967191170741,0.7262349553536211,3.1537076356699982,3.373339403490877,-3.1752935301163974,0 +-1.7908857605968178,1.962991490408586,1.7314129963548224,1.6432135637840317,0.3502205399270687,1 +-0.14668617091136427,1.3119507709575815,0.41341434643023267,0.024506807304796152,0.4295722816627525,0 +-0.04105530194044826,0.8745682041764067,0.4437974500929688,0.3377787156869606,0.23000896568607554,0 +0.4380154618528218,2.1632169614454417,2.1895079170200074,1.787313589024771,-1.1037320271478652,0 +0.1344092807008681,1.60141864414982,-0.30645978194555795,-0.501969318636718,1.7203261603367386,1 +-3.531215401596665,-3.6234502459925295,-1.3780734885059267,2.1054900049311076,3.0768351065421387,1 +-0.971076609785853,0.8018095667984867,1.304180230589901,0.9247011345107701,-0.8861363521048304,0 +-0.6762560123708579,1.7272392482678094,1.2059859529651245,0.5784361940199746,-0.21026189185262933,0 +-1.2128773938785697,1.141319253300938,2.141512130136142,1.449518242045872,-1.8346490890558997,0 +-0.6698391409513256,-0.6761049501968288,0.027968404001009306,1.2312152269518322,0.8377602400407796,1 +-1.8027067813783524,-2.7112839663166106,-1.7686886739636862,0.3300618579880855,2.364902567367835,1 +-1.0742081535420418,1.291694091996956,1.4732860133934313,1.0683013164183963,-0.595731790957813,0 +-2.7796706886187543,5.626749333286726,1.8624426378243188,-1.3185545711676787,0.7880570612175586,0 +0.7257848927633654,1.060579895499629,0.6906016934274193,0.9583113036697242,0.26844616365874074,1 +-2.37522766822483,-2.0451239287964667,-1.180481853099383,0.9666969445196764,2.557449004775004,1 +2.690019885753384,0.6775412615597949,2.6847603167114453,2.99840395470338,-3.166826566555012,0 +-1.4527241148341803,0.7968026108250906,2.236030456774712,2.1902061748677717,-1.5037450523331952,0 +1.4994989967196761,-1.0098105059361089,0.34466268120099847,0.5272309664379385,-1.7364317785011982,1 +0.6572318193277807,-0.06657889226576374,0.6566714533282095,0.6161774089445622,-1.1362727778301251,1 +2.5526070115695867,1.434105568343269,0.6120081943422102,0.4466609188622269,-0.3505430941887021,0 +-1.0102227607100855,3.8357361941069668,0.8541760269380676,-0.7615969382823575,1.358068519138766,0 +1.4968746279453529,2.1978874127509824,-0.24658097769279813,-0.8971842255940903,1.2866762789672306,0 +-1.2204179063074878,3.7559279914904526,0.7976902493252214,-0.8654861309628465,1.365573897360882,0 +-1.3622610294318305,-0.9652773047207448,2.3255240188450084,2.4333614751372403,-3.2408358149506107,0 +-1.2947104671925396,0.9207032503037553,1.64406253771807,1.515803670948587,-0.7981234866924097,0 +-0.8085505608806088,-0.1892958642687469,-0.3400889428238974,-0.6728241900217808,0.13168664331508773,1 +1.121501327537658,1.8062239412293561,1.7772074905372004,1.5696760721652059,-0.9713863321345013,0 +0.17342903105151497,-1.5638837150495446,1.5275223291005042,3.1513042076572413,-1.7089906077550148,0 +1.5313614732764695,0.18156574216197297,0.2561231404380712,0.3716760466853982,-0.5182040967414809,1 +0.21948162034841334,-2.933230222311794,-0.36554571593541785,3.0736681852182146,1.2212992318985685,1 +-2.861360456902147,-3.2299395182719293,-1.1379726327426374,2.062987743943878,2.6412381908510065,1 +-2.109413225945374,-3.3757824081111143,-0.8479795833621844,2.649459127728171,2.214036156046829,0 +-1.0530647535257225,-2.1294462941111147,0.6942059245675882,4.299072791114557,1.3380343909750676,1 +-2.3361131705204916,1.2954641994176272,2.4866719270583455,1.4367887996848725,-2.1203186325089938,0 +-1.3903924477941891,-0.753803126614216,1.949271226645867,2.272329554281728,-2.3214660221422356,0 +-1.5122537883904568,-2.3982689049770536,-1.9047277520058847,-0.8553376993292074,1.5923015121584692,1 +-0.8318832527987848,-4.105529463987493,-0.769813584835227,3.2052302641957864,1.4841873408083464,1 +-2.0152050241982553,0.7288340027981387,2.6667886668487975,2.1011355949749184,-2.48336527287508,0 +-2.3058656067775,-0.5741407341886278,0.12052574062527144,1.129960945445374,1.161944274485102,1 +-0.45460551697253593,-2.80532143251547,-0.7898107182456211,1.463011029253518,0.7888908515133407,1 +-2.2643448119278435,1.6531811584090434,2.767888112181035,3.4834579077327943,-0.18613777985865676,1 +-0.8674772145052044,0.48255432954590427,-0.30785438404770377,-1.062151352077517,0.32119222893882404,1 +-1.5773109254432363,1.339890715494202,1.0874775359442197,2.064653624178626,1.6160816100117374,1 +-2.88613063253072,-1.0434874909692404,-0.5363202674720657,1.3060120116425085,2.608021551445627,1 +0.03957915492683073,-0.21971374168436708,1.0415764893278059,1.641829165282114,-0.8485110205941212,0 +-1.0637078225626546,-0.9667530804038879,-0.5761125228040718,0.2643142252166729,1.0190276833369347,1 +-1.5595711052631271,-1.6772783128100204,-0.3146247767816769,1.364118146168146,1.0759692755933665,1 +-1.136478144714418,2.419814644058498,1.543779203507345,0.7408950554333074,0.03145343826710767,1 +-0.8205745041335408,-0.982053132593995,-1.4691048243451759,-0.6253774406949286,2.019753799360877,1 +-1.1857430705437462,-1.0623088029381738,-1.970574375811211,-2.5748608130410533,1.0815272704648806,1 +-1.379015157677912,1.1178061152315832,1.6845389211075799,0.946677547877089,-1.2931716001869022,1 +2.572429009456915,1.1142875159383154,2.1375852964114888,2.1719925785418566,-2.3267559907882616,0 +0.4028448243112941,0.9425994976380745,0.4769270905889601,-0.19779640288418232,-0.5175670266140191,1 +-1.4332927484257558,2.0120055446450653,1.337798930892697,0.6091843290501067,0.05635715673132058,0 +-0.14539644945775265,1.8655295512421206,2.562932040718172,2.281804355491441,-1.5305848226559344,0 +-1.717651649632793,-0.7375639036860908,0.32641837991317146,1.3327173769479161,0.5451251457059999,1 +-3.152258329639155,-0.19674316307229933,0.6347709775079505,1.7755376028507612,1.3388063153322785,1 +1.9092356577251914,1.9958364611985928,1.8881797725067877,1.7493260913261173,-1.105877625179031,1 +0.3179884580855621,-2.4048778247132736,1.0824509354712046,3.246135813038887,-1.4606929575100356,0 +-2.063139220279875,0.42769620153913235,-0.24178015079019016,-0.1424800505201429,1.5263504119959772,1 +-0.7403711983761396,-1.5927228992246443,-0.3340129318379125,1.6811796640410637,1.280195786159192,1 +-2.06040179119844,1.7007383565108016,2.222804508868918,1.2348747048880133,-1.4150697340673328,0 +1.3943913171483058,1.246451795287605,2.419991273046495,2.4129046680759547,-2.1907704067400604,0 +-1.4826136781476222,0.9230354839139653,1.9188512048171913,1.2368586024787063,-1.6791861944469106,0 +-2.6750435362836176,-1.8623069413686126,-1.5037151724206077,-0.2361262854389128,2.2686248812935266,1 +-1.9694635529084281,0.49552056984938875,2.9951594061267306,2.230685915584953,-3.353814028255917,0 +0.4611114255906521,2.5270872047282245,1.5419623179103352,1.437812203696176,0.37478072681730634,1 +0.36052150400120997,-1.5203229042214872,0.9951311330416465,3.735087347281806,0.15331349479883782,1 +-0.7452779705850647,-2.9258006287704643,2.3890331159170364,4.00591269392447,-3.8290342696432953,0 +-0.45656094698280936,-0.1483653837949197,-0.1646146694203282,0.5531206166146156,0.9964932712357841,1 +-1.235461117507949,-2.761577006228793,-1.1484067003556238,1.1244714956040265,1.5560704088307729,1 +-1.351484872683368,0.43701810779519235,1.45516973437466,2.3239244116089353,0.06543042383589914,1 +-1.7199946846736243,1.6339616676306794,2.1525345458300276,1.1871324935055285,-1.4853005914978221,1 +0.38608164752349083,-0.07957019482201977,2.778026482010281,3.0805066951898312,-3.2794698725818083,0 +-1.0587462275869144,-1.004076780455612,-0.13566076338713534,-0.28124489116693474,-0.6454735583870557,0 +0.6670108775735097,2.4671085051887003,0.1748656144057161,-0.2687193507166836,1.5465916374573307,1 +-1.2515715945591679,-1.6465656481572286,-0.36269109458383497,1.715179142795547,1.5018591616966361,1 +0.15050089846987857,1.2499037865346743,0.9018210361699831,0.3402308525011686,-0.5208161568550818,0 +-1.1496876451863487,-0.25442506678141297,-0.9836589185392473,-0.6434370120955206,1.7103449677072624,0 +-0.16271542793796578,-2.065667146577949,1.172153676818231,1.9705198524463863,-2.5728518868560215,0 +1.183138995519342,1.6743658240105193,0.83169169775719,0.23029764074905246,-0.3996441901014067,1 +-1.273794446946026,0.25517795300171797,0.33602828852720007,1.0483685959607287,1.0558780092903277,1 +-3.360157107831941,-0.04950293085677726,-0.34250692420913365,0.03438003030212955,1.9121676446377447,1 +-3.564583678249846,0.47444017103475644,-0.6360363245064411,-0.04501945278175179,3.1013527975513697,1 +-3.7784196196501045,0.2694647216635415,2.46953120938777,3.4654500265065886,-0.3910578388925614,1 +0.5637407316611003,2.624447540182201,1.6181928108933197,1.1186386025320474,-0.09087411739607654,0 +-1.8196171960825556,0.3012193942357726,-0.7290630642425491,-1.2913599843199592,1.1862631732325915,0 +0.8383080115370747,1.6300557123781565,2.104662598135927,1.938256910094327,-1.4088524213429134,0 +3.460292538043455,0.8811741641538542,1.8944403205289482,1.9113624447946593,-2.5796443673094167,1 +-0.3719561952346768,-1.6142180651774543,1.2372118948191038,1.666356517710417,-2.5364289779941496,0 +-0.021968570557738487,0.8209752200973923,0.5953278003654234,0.7911463808000105,0.3166430353087514,1 +-1.1334665139825209,1.2153442070374183,1.5894985996841997,2.583588076060166,0.7459596195678235,0 +-0.02737119086763795,0.31469908268337266,-0.6271327798146641,-1.26947493860989,0.38555325994272605,1 +-0.17159053229648003,1.4941483065567072,1.295940052640618,1.1944575434668279,-0.1440766767230286,1 +0.527995330559287,-0.07394530719157988,0.9302634630519093,1.6045994053752137,-0.6482773672304578,1 +-0.09754373362922242,-1.1747702929428674,1.101249174799772,3.87435896100766,0.5598618792238925,0 +-0.44411016849247675,-0.05669341065013067,0.01821745918643647,0.6781871885456677,0.7966120768526541,1 +-4.180095994293889,-0.7519047170165374,-1.2142746936556486,-1.1130945805825228,2.2474823316624355,1 +2.0024243881555295,1.6732656324090316,2.4429970751568937,2.4034966388920793,-2.030768935507846,1 +0.3513510294616805,0.7891161198688877,1.9736342878049118,1.9384680515887018,-1.7834253625605383,0 +0.9262855120214887,2.6726619219748464,1.5239487897620083,0.9283315829451703,-0.15448951882038642,0 +-1.6511470416453706,2.92370020207699,1.424848031471328,-0.5591266832127899,-0.44576778019144137,0 +-0.6722954724120327,1.4278133661393675,1.1521018096303366,0.14141151159101517,-0.8676555485197377,0 +-2.665141848123402,2.0562133105704077,1.9143062769862578,2.3980684537260855,1.1427406009017274,1 +-1.4331170979667427,-0.5403086002926527,2.485610444185252,2.9966523255075046,-2.5431679750260647,0 +0.6701141668516459,-0.0788636586975866,0.2864111227706914,0.18626362165384935,-0.7650030967087256,0 +-0.3377963372494488,1.1649050168117467,-0.3872622612113061,-0.11829737091183556,2.051646927287605,1 +0.9980199922471478,0.18559842413835825,-2.2407553155277187,-2.3651008048176574,2.459432866402785,1 +-0.7945504510619537,0.5175828283290962,1.4919501326537121,1.7129151191849186,-0.799062735588009,0 +1.6503342945850363,1.1281449359126885,-0.072040078523959,-0.2750845754297344,0.4419213912638923,1 +-1.2951254681450182,1.721086368160066,0.971925648405922,1.0581178262071773,1.0639110393743985,1 +-4.699895153393166,-3.867973737991636,-1.8321300147720707,2.0036382778827204,4.16786757311883,1 +-1.1635201700965954,0.22842715159760463,0.6933303113526106,1.181074335702699,0.3070140224080816,1 +-1.7088349058434458,2.2649026657377282,2.021274501544377,2.3899689942894033,0.7733826518272507,1 +-0.34544661643434693,1.9234084399442577,2.4622212341544674,2.873721276863577,-0.5193322150078987,1 +-1.3042794657909509,-1.9550651965776997,0.5199582405166336,1.909565958034472,-0.6289610858816155,1 +0.7255006286327386,0.3407928940325726,1.0324980121790721,0.9582394731284781,-1.248419594612733,0 +-0.14759199836789938,-0.5671728353625842,-0.9164920709392783,-0.4243330224333772,1.1457297707141836,1 +0.4664713114853041,-0.5977920411016153,-0.1604283701873498,0.9089816340225345,0.6185064145432544,1 +1.8470474648833464,-0.652897169715829,0.8518165836107418,2.366979107005586,-0.6490386453174093,1 +1.1070375198014373,0.9188974988525599,1.3514320157305821,1.2326130682337857,-1.2366973681968387,0 +0.7844427257243916,1.2343393135935017,0.3180708736184256,0.24555627630196764,0.5046589795595046,1 +-3.230743460828335,2.645890246031403,3.028258154743457,1.4390742877901528,-1.7239237986626987,0 +-3.3310269275432423,-1.0790975803309828,0.7408894738587505,2.2629192947970895,0.8041510592522246,1 +-0.5071823001061533,-0.3990231852180639,-0.522464175290032,-0.6059388750224177,0.318281116896327,1 +0.8926111097712255,-0.4926592574631189,-0.15765732017384235,0.7393700154232805,0.38771652763166065,1 +-1.4898899320281225,-0.9844451125575433,-0.3004114525788173,1.0225108949735973,1.33784180364194,0 +-2.3819202395799417,0.4210664657169625,1.3206598572243011,2.5614231708184123,0.9695511398639456,1 +3.37855832395024,0.8631560091773784,1.0671389074706847,0.9886777063604208,-1.6618189354718693,0 +-0.08540395263657086,0.8020044423531687,1.5844536471039818,1.4700369574009233,-1.234495344396578,0 +1.3804882134633842,2.002848519678997,1.2883336381663275,0.9731013395471988,-0.3808591751124092,0 +0.1862917031344994,-0.12367742278498728,1.5384332037843644,1.6448577560066886,-1.9539694787287054,0 +1.4699335556270825,0.3410673459145508,1.4848480122784058,1.5068824896797874,-1.9459837128276858,0 +-1.3530471723386521,3.6821141057168965,1.0172874178173004,-1.0785584112881903,0.5905749807990572,0 +-3.3980736760193637,-1.389051460988623,-0.20957081593546523,1.5910454119110038,1.9871201895351,1 +-2.39805677797749,2.2279537310758175,0.06492822802247966,-0.1742081358595915,2.6948518061455653,1 +-0.4036034348858124,-0.3565177829855233,1.3140368866206753,1.4557870779149575,-1.6756883060783845,0 +-2.7179989931840245,2.3610511176102724,2.9366865615005495,1.3809659309112254,-2.032413674659284,0 +-0.953012135089474,1.3583173074168262,1.779729866784015,1.1490512672810336,-1.1935638060717118,0 +0.430195940253815,0.7142321002575143,1.5696382220462068,1.5481064023401832,-1.3749961402029962,0 +-1.8926922186910788,-2.255974293880211,-0.8511751449004215,1.0503662710723807,1.5102936352700345,1 +-0.7830025832324586,3.2773777708800758,0.553044030554017,-1.2513888280910637,0.8832949269426627,0 +1.147437207575659,-0.15164732938544145,2.0181749187803604,2.3542375196646956,-2.6392533166627032,0 +-1.3992608764550365,-0.09225318455408793,-0.17773450173630467,-0.08506670142183337,0.6966881883722482,1 +1.7404428775060694,-0.2771524063608892,-0.18802159957105788,-0.34466159595783297,-0.8038006864448825,0 +-1.5158878762156949,1.8964688901442628,1.6487799888862409,0.5728727916781783,-0.7951808583902484,0 +-0.4793989467865878,1.8661102324063834,1.7915363722732964,1.8187619438279796,-0.13337145365765246,1 +0.11986738891266091,-1.1287676859174192,0.964372946980242,1.8512983982440667,-1.3764123411869347,0 +-1.3738662161152317,1.6951663592491766,1.331152261674233,0.2555574969705733,-0.6551459518107414,0 +-1.170378728125851,-0.7318267017985933,0.5396322353868148,2.584179224688855,1.249452883440511,1 +1.281996261879405,1.1903118102593129,0.2842160119974789,-0.04960090040356091,0.04734160685501698,0 +-1.0224528297890734,4.696327212328067,1.7222354592432838,0.7625664477006322,1.8839231436927037,1 +-1.8472825028842903,-0.21762968968901575,1.1271224228707393,2.58009968651018,0.6207710418428616,1 +0.6907254174496904,1.980508959224327,0.7376613699446846,0.6799348242000365,0.7863178631496419,1 +-0.8842128526730392,1.0634640404465578,1.7535956357631417,1.0341805397646984,-1.5779098849594284,0 +-0.7973826752304377,1.7859522340290273,1.823810438965471,1.8521982385260634,-0.1453149260115334,0 +-0.2826395282944725,0.09859698495928815,1.0711407225715162,1.3456531607446796,-0.8162831572186477,0 +-0.3084515798795451,1.0690206942719902,0.9100414468838657,1.0495519036731304,0.21382787328831077,0 +-1.1355309520352503,1.4674397550313394,1.7048894822997167,1.5646992323412887,-0.39112229310443747,0 +0.715393053765699,0.5343964735186322,-0.904369800960031,-1.0109373797242265,1.286277354116597,1 +-0.9861941779691142,0.2686171155894226,2.133891268753648,2.2528525527257797,-1.8831018118826202,0 +-0.7558609272896826,-0.3834623388280274,1.1126521037018748,0.9291113939934403,-1.696045911082663,1 +-0.7426104769036281,0.3158720449654637,0.9244096865280529,0.4601046969261883,-1.0780477985679184,0 +0.19579630074165666,0.0009451448809769669,1.8862833729451294,1.9774667394929257,-2.275162963612118,0 +-1.3866351433028041,-0.1775954601401939,2.361079922530949,2.8332721006203387,-2.0853958415458056,0 +-2.3939927362949995,-4.340499409828883,-2.184367115497969,0.623442260481649,2.2201873685565245,0 +-2.072527784456982,2.067309692939296,0.8302657224862183,0.8935016750551722,1.820009079736349,1 +0.6600780684137897,-2.125385412574427,0.5988008440250892,2.2359918077627396,-1.2825659644655572,0 +-1.825703045131993,0.11745027426142596,-0.6943244047738484,-0.19797047267987855,2.127364051411047,1 +-1.437833344864777,-1.509161496878043,0.45190534603338706,3.341503984088233,1.598038190654035,1 +2.6245648117759495,1.738039793433062,-0.27670534494328836,-0.8404467084192622,0.5800025045465871,0 +-1.1029519523743057,1.2736497747866429,1.168507592794759,1.516147152872163,0.5980202645257587,1 +1.0851713179196727,-0.8940644829880732,0.25409267691875337,1.5353133959537153,-0.1605866611723814,1 +1.9273121596147358,2.1300439481565157,1.6156534052497245,1.2597263026138057,-0.8825940008517918,0 +0.5479971926153351,1.611530174414562,-1.776851225392906,-2.6645291202832464,2.6308279745600247,1 +0.639926019310195,-1.9996095478109273,1.1278844023002699,2.7625881544574193,-1.8006352295592567,0 +2.1865827375994966,-0.006849517706070785,0.3419925712784925,0.3621522679659537,-1.1371120561820225,0 +-1.2352605144849562,0.2745363641848424,3.084890003129212,3.476508657470033,-2.658407502092991,0 +0.6571720851343019,1.468817091210764,2.0427483319540443,1.9707915331831325,-1.3298577000289156,0 +-0.4316822231372386,-2.1957079299533353,-1.6349398459031432,-1.268592272553207,0.3501726644327826,1 +2.210268547371121,2.0023117800906043,2.4050951668259164,2.3178060151384043,-1.7768440285459701,0 +0.8922197579307974,-2.7758127626357707,-0.13094048489670956,0.3573819070766706,-2.3836215699559444,0 +1.1064405159504629,-0.3840921824342858,-0.14095146974397554,-0.23359848205849776,-0.6855181255769481,0 +-0.14518075720736567,-0.8175471658002196,1.5609818803478932,2.133649146440424,-2.0524994453832996,0 +-3.0644249018002396,-1.1087104854524163,-0.5509751235502053,0.592151467803766,1.850409117189069,1 +-0.03804872636518175,0.236868710669764,-0.27106103611700094,-0.8064002285970302,-0.007263030613972266,1 +-0.18427236611124376,-0.7727342178518397,2.084737836554071,2.2322528984432424,-3.104302623032173,0 +-0.45304295415602946,2.885506207814581,0.7397198829925555,0.5618894458297679,1.942383041963169,1 +-1.3020500920184763,-0.8121370027867627,2.272521488797995,2.33642578704117,-3.090972539545532,0 +0.40604455225384317,1.866917035870034,-0.3228644415023857,-0.5347473807078809,1.8974116954692772,0 +0.14950397418466688,4.315398480194434,0.9374393206917495,-0.09687077769368413,1.9881884393786664,1 +-0.18142672451533515,-0.7450424847562123,1.525748430562944,1.7531614396400332,-2.304705637817097,0 +-1.6103531561158335,0.34517283649544117,0.6395365554618206,1.2302743007532913,0.7532515540173866,1 +-2.6522815459157147,-2.6192061099048214,-0.9065792709661638,1.6444366533021086,2.1840889183822902,1 +1.3720256775654263,1.1766746161370232,0.2793159818330495,-0.014060019360358611,0.05400969285935742,0 +-0.5221182946265343,-2.4327349579154673,-0.5113692988648673,1.2960015295108884,0.3533609376994866,1 +-2.0152667861246716,0.4057320307490234,0.9208210776206208,0.706085293876693,-0.2818326993341542,0 +-0.6354085171375272,1.4872379682098882,0.11290070766951468,-0.5519271292728085,0.8335199558102054,1 +0.08151836346102226,0.5971028523175346,1.8921982281223508,1.8678793416914101,-1.7736762191643702,0 +2.1890654811427477,-0.8559419336067196,1.5373306010530063,1.9354903115384956,-3.037198623555004,0 +-1.1421652143495917,0.7126294295398583,1.958633687268007,2.2468010687592415,-0.9846585409573603,0 +0.42056496129497434,0.6077443489230427,1.279508535138206,1.0710704445248804,-1.3289068326792008,0 +-0.2741334290683849,-0.29617504495605784,1.2265628479410087,1.621924733037381,-1.272180161427611,0 +-3.1952380625774794,-0.3384360944301108,-0.18241922331834926,0.9953245805587717,2.2530325624622654,1 +-3.3374933326669134,1.729835890338284,2.827853264146113,3.9972984492986425,0.6770481400283077,1 +0.6563819780401157,3.213813638896659,3.473594037748276,3.4079701408041196,-1.326315092594193,1 +-1.3554056309101163,-0.3508671887713406,2.68805675893081,3.2690385190737494,-2.5503300020622603,0 +-0.21023058406542605,-0.1974380049698623,1.0825160849850666,0.9521892749728701,-1.5970819883753347,0 +-1.2516282309921252,2.4529504728183804,0.7027205930687741,0.3716892207766668,1.6532823124091673,0 +-0.7850626346700532,0.5945250975698122,1.670024673002114,2.4740763184906815,-0.30115445641480587,1 +-0.9999878637054567,0.09526884694013305,-0.2500755626644043,-0.12004413558434068,0.8804978405757139,1 +0.31512629112183466,-0.6383682723118975,0.8483766728534774,2.2498799033832344,-0.24155110927032108,1 +-2.8653361653609215,-0.32433917505990517,0.45031211737780585,0.3222668774170927,-0.056634118717002635,0 +-0.5947029314404539,-0.7672570851390981,-0.3944608502790167,0.4350227528354056,0.8270903875758379,1 +-0.13609895481961676,2.6577948524744572,1.7664622562818162,1.528441658108402,0.28368104771272185,1 +-3.1635478303235485,-2.44051158074097,-1.9624254902183873,-0.15429922452392608,3.0101109958355274,1 +-2.931589374516034,0.7630246425679839,2.8110573188590706,1.6585207502575812,-2.964735774782981,0 +-1.7731933680370298,0.7382986132046683,2.12999687874008,1.9020113496553541,-1.5255939898621205,0 +0.848605734550433,0.053340709386014096,0.9129278568227747,1.3904254099771445,-0.8237606750362854,1 +-2.077749694981592,1.2060838415611133,2.529500051720093,1.7599159195584915,-2.0403724890298385,0 +-1.1188689268961265,-0.20285904972194568,0.794779276879819,2.7243104562149902,1.3231814713383816,1 +-0.0954175913191071,0.26825425209160625,0.7833150023168168,1.155335397728294,-0.2490306969852848,1 +2.5390054121343146,1.6099634285770086,0.8634401458207299,0.5942047829519292,-0.5921285368225542,0 +-1.4794498803961207,-1.9944796471316764,2.6921534412649595,3.4204451955460318,-3.9990891618057414,0 +1.2774871040329563,0.3184220525939194,-0.6303355119697646,-1.0548169993328025,0.1954172762731774,0 +-1.5311024724204771,3.015082305734303,0.0320953116205837,-0.461751926828889,2.952786971687642,1 +0.7087423407483889,2.117191185543434,0.5234616899314095,0.007205425183772318,0.6747459452722488,0 +0.00047988707795476593,-0.9687107196567223,-0.23968485234385523,0.7714105907167034,0.43588539521221004,0 +-0.0161488106623352,4.669858118845295,3.1210033665177224,2.17567235260933,-0.17798132744404005,1 +-0.4733574009373662,-1.5865198933906508,0.7566849118259822,1.052750114602189,-2.032518381890227,1 +2.915132744938017,1.5785145276602148,1.1804433268067656,0.9325964531999318,-1.1146234531812436,0 +0.10093815568700482,0.8406918011448388,2.1656653730016506,2.1528204483240563,-1.8583976901061368,0 +0.26660966057789126,-0.5327505054580786,1.6533728603776932,1.9365706768947253,-2.337111160745244,0 +-1.306766266794749,1.3721033307752952,1.9451614090689782,1.7865535695818553,-0.7438827417376146,0 +0.2129870676981067,-1.7808155165097936,-0.204253218019261,-0.15503541668157306,-1.5521264634547975,0 +-1.8387596077625608,3.4837695802186133,1.7645980307905438,-0.353269844830417,-0.38508385382087695,0 +-0.38110704059262157,0.5458493726301588,1.1278493163692698,0.7499770883061763,-1.1228271281145976,0 +-1.8580951731542594,-2.2307726004159716,-1.5009570089353417,0.00166573101847578,1.8806794525585093,1 +-0.4437113239703483,2.4614639808671477,1.7825145718053808,1.9602162609219822,0.6282088714915346,1 +-0.7844587082983987,2.083342056980249,0.9435954528530927,0.06979558617981219,0.23326387485465294,0 +0.38226677817715304,-0.006200059600729668,2.690762213998934,2.9751146425417647,-3.117700630118287,0 +-0.5309278522168592,5.066865843663481,-0.4661101062627375,-3.1374154970101413,2.8864483947947432,0 +-1.410019303549808,2.4696401250342053,1.3716439193152965,0.31234784762704115,0.10166525901384227,0 +-2.968130722244831,-1.0600420652531284,-0.15237497138197664,1.0800766359299163,1.4769450395539705,1 +-0.8182210711725721,2.5262596157630908,0.5077096281967457,0.16726334750541338,1.8099195616385941,1 +-0.3433153061155022,-0.2553739382456953,0.3278599093016218,0.989505894495173,0.18615097927577462,1 +1.064763447816254,0.6388726488478786,-0.6353496895648167,-0.36060989272132415,1.364150901970055,0 +-0.8627510786187689,2.73870902356454,1.8694695233951253,1.3650012316407736,0.1889272358993046,0 +-1.5144949861995352,0.9464555442313377,1.6552408779580707,0.5418574089732152,-1.797136808573561,0 +0.7872459278367461,-2.3465109064283376,-0.17970963188890554,2.3866044211085935,0.4300424548141356,1 +-1.368486486296126,-0.8298239807292969,-1.5983665977551478,-0.8203610142958142,2.442124734289842,1 +-2.1900802833863624,-2.705222886595946,-1.4998309254692441,0.3575253271372084,1.9057157234870756,1 +-0.5944304413141811,0.07648106833299984,-0.5128673803000896,-0.01505388520269535,1.4525796937476831,1 +2.205311501013434,2.87764561316981,1.1924468989377601,0.6428837203925529,0.07877766318581303,0 +-1.1694690187474925,-1.6922218515468277,-0.5413061895205908,1.7843306832511756,1.920220232124858,1 +0.02169045265202829,0.28577081533522497,0.57274304642899,0.5174921281254792,-0.48331638566901236,0 +-2.1262731706614,-1.4908729720098677,-0.20213130075300634,1.6528753743755034,1.5092167931068188,1 +-0.5166990351338983,1.1125629463304683,0.8392744352190076,0.3207341333975612,-0.3106461994135301,0 +-1.9018373806185203,1.0130998539124594,2.740807673679642,1.9705143780720435,-2.552480088833656,0 +0.2746188684647839,1.352856566280071,0.1626158380649908,-0.15765320572363017,0.7120873904467662,1 +-4.004869310053971,3.1486777336534466,1.765675716827526,1.8344939549015717,2.412374619697631,0 +-1.1602302676592828,-0.5614874171348887,1.8944876428836814,2.2391469135236113,-2.1149915158449666,0 +0.06459193737589453,1.3638540025793289,2.212482559165283,2.4090888558127466,-1.148480917733877,1 +-1.593087451627724,2.7744621112437753,0.3555648732548897,-0.22875078352965916,2.2364760973958617,1 +-3.128584378019733,-1.0911466013197306,-0.4956850484555142,1.1516239114172049,2.3770286058830763,1 +-1.6982642305882938,0.013738420719339839,0.2870613090244032,0.8021998227489997,0.7990647255364014,1 +0.24768995621618872,-3.6934560531095864,1.162625432150056,3.7674016087104385,-2.3412683756059005,1 +1.8122794244527891,1.2840765729053571,1.5737684484570966,1.4381080921515934,-1.3981342630955338,0 +-2.056981582377505,1.000746241417191,1.159502794899801,1.8753955225019447,1.061050481965919,1 +-1.2661610848654723,1.4731728054260682,1.6357169743202529,1.1128825402142797,-0.6781976103932238,0 +-1.3668951333908024,1.2916832621812082,1.4534160095360353,0.20404110076973103,-1.4030964798775367,0 +2.266331755156444,1.4100671777467448,-0.9897002915160751,-1.6749763110319038,1.11104323108902,0 +0.08521829392459668,0.48582146865581166,1.0359831897554665,1.7983239178119481,0.02876550234356756,0 +-0.23818963674681393,1.2416373485393306,1.2963885061556009,1.627663364110775,0.10116457698095749,1 +0.8892306496313445,3.397805282183613,2.0878568468383842,1.3990176426074203,-0.20866019982996742,0 +-2.3794062934004385,-3.0696846568674827,0.3665201039187584,0.6219533385284765,-2.4466459256613127,0 +-1.181088819932628,0.6550058679807468,2.6798044816311126,3.7145470996426213,-1.0906966858826856,1 +0.10180056002122373,2.4380012669868725,1.7698332048950611,1.4539894463346454,-0.10627626682031432,1 +-0.830728129885048,1.5652155463439628,-0.04665880114874221,0.10561167914707081,2.0724992100898847,1 +0.4053469140716852,0.5085149521854901,0.12557040548966208,0.1664542083900059,0.2649581900443695,1 +-2.342044763600879,-1.6102245339139136,-0.9009841600558268,0.7439656217456587,2.0867839688512593,1 +-1.943594805398887,0.6026664336902904,2.149408914681342,1.8587993688549305,-1.6971381180693061,0 +-0.22145061426608326,-1.3802511151583472,0.8970257164432489,1.2003864539383198,-2.0741477948296745,0 +-1.1107140746496205,1.398575537620811,1.594169886120472,1.375703947841755,-0.41903729343637486,1 +-0.7501284788110372,-0.9460285041440755,-0.006853932826404896,1.1213081086011598,0.5540924679653598,1 +-2.222054396135138,2.592358408534092,2.39211361201816,0.8777125969910466,-1.2542633552423696,0 +-1.616243234686673,0.9938414758984357,0.7773390100652748,1.1467664435297087,0.9929247654478617,1 +-1.7244738756261175,0.5295888741918227,1.1348698041257732,1.6026990693206526,0.23421918111137052,1 +-1.3397535333995925,1.4524301538941007,1.5675697451738215,0.5166817054168771,-1.1723322419562998,0 +-0.4856852714574713,0.18691306601596092,-1.4252170959616148,-0.7689693778204301,2.8189745204539434,1 +-1.0478188489343447,0.24228945360911802,1.5879043108308486,1.4141583763716645,-1.5422850563431323,0 +-0.3026220797896213,-1.733485642705193,0.6392241515257823,0.9743323085100991,-2.0502163270292275,0 +-2.183849983904972,-0.3101082258850044,0.2771304783973557,1.7368409938398748,1.689691815913494,1 +-1.020139206182745,0.13171892378542494,1.4722186987595505,1.4041244027433903,-1.4043352332824608,0 +0.6013660904301237,2.8004285616392037,1.4205178137451582,0.7848773410218395,0.16521364314630993,1 +0.7634360370600045,2.0248846181685733,2.1311332107291685,1.8295655782622036,-1.1692307718720498,0 +0.47307641905248743,2.353525104688271,0.5460729612318432,0.027616086347507784,0.9605785312823023,0 +1.1053304543028952,1.8893127422602787,2.162096339685987,1.8783357865679275,-1.438138685322984,0 +-0.7138213077743706,1.7869040529123357,1.4205822780176687,1.5941940893394366,0.48152184816815435,1 +-0.8891323424498941,-0.36308613733585393,1.1333748994243873,2.778566473852072,0.3575714690392612,1 +-0.5547510768756081,0.1081850992932003,-0.6733035177255997,0.01589911541387834,1.87836229164553,1 +-0.34186297013162203,0.2297043108749648,0.10820640206396333,-0.13168373794546073,-0.05180382231499925,1 +-1.1907899668219804,0.34244813705098043,-0.14796631363379606,-0.1862098889597057,0.8818734473676779,1 +-1.876489014446142,-0.02316286704502568,2.690727492332873,2.6942024583583617,-2.6862865240547755,0 +1.7285088360239254,-1.6487589150274347,0.2842422345853489,2.2321338847827787,-0.4356753214496414,1 +0.6651067710198211,0.30819928604673197,-0.2512765521375714,-0.898994363670609,-0.3196241166145162,1 +1.9782040375531553,2.628504187663925,-1.6040206174001896,-2.6537904721434713,2.7802638230733763,0 +-2.6510828902659993,2.326386801219159,2.7204071772018747,1.115068251448843,-1.879290559542157,0 +-1.186834146569784,0.400025501518365,1.7401439218567984,1.2164498335057194,-1.909419390148308,0 +-1.721182046850616,1.8328720498254603,1.1220953073737008,1.1403298533081758,1.06007742179251,1 +-1.0976449177973853,-0.5588748375335651,-0.11428676794601245,1.046114122273532,1.2258684905247472,1 +-2.055258399352235,0.7297352574254683,0.25219838395878286,0.5767320764466431,1.469347481897939,1 +0.9396325057306661,1.8240610018996333,0.33292806973467104,-0.12167131419750987,0.6047759997645985,0 +-0.09091993774665363,1.2264744367918556,1.5612489629809438,2.1151270225387098,-0.04277116280580451,1 +-2.0128888578762956,0.9541551446895393,0.2412095201108238,-0.018599588426386093,1.0500559717062075,1 +2.381806116417988,0.4540864286745504,1.4210397807376298,1.4490533405462178,-2.0538209842831057,0 +2.160066151875852,-3.0363564548476267,-0.9412889899282006,-0.5780084656984401,-2.2150464752320014,0 +-2.29951734855831,0.7159332151564084,0.1063868128596227,0.37019985974029024,1.6492103469246384,1 +-2.1169951863111063,-1.784911689560245,-1.1179894043374845,0.557798912758207,2.1359931651711097,1 +-0.9340102532402836,0.6630945859933702,1.6881548302857148,1.4345176531864965,-1.3694364557519196,0 +-1.4213411477185902,1.7459448795822081,1.3604699327788818,-0.2257296245975493,-1.186534886247403,0 +-1.7296842285076708,1.4741464200540437,0.20821073658789635,0.48095677044851903,2.1031983283215974,1 +2.5425864183505036,4.092218706225297,2.232066623101205,1.6319321340706483,-0.14616646226927688,0 +-1.6608127328114428,-0.978266648347326,-0.7344388104076814,0.29683572506910316,1.6119138738266148,1 +-0.9874175244628917,1.0977031383960472,1.3314180616152052,0.05653674473840642,-1.603469679402069,0 +-1.3501854845298977,2.3902491287641086,0.19801646149880547,-0.22970689511433728,2.1353117390007395,1 +-1.2824786246923263,0.8626410830246449,1.6330904953313787,1.2824759378933648,-1.091854168122208,0 +1.2087260529574664,0.9001894851085859,1.9538468070830133,3.065143434466759,-0.6729141903713469,1 +-0.623248462831707,-0.08381826763728495,1.0679325456724942,0.9470072258302213,-1.316424216195174,0 +1.7931884976559327,0.12649432032333552,1.5489265183842555,2.974289255357829,-0.8026077354330869,1 +-0.16763373601916265,-3.1031377831109586,-2.0402743754390076,-0.40379116317383734,1.2472732826269155,1 +-2.2159511427157663,-0.02821050295166927,-0.7730512531149942,-0.13380289603564655,2.366066770059766,1 +-2.4999385413196746,-1.0101783866621894,-0.5480421839854156,0.7704605091578789,1.9493626188357425,1 +0.03854326291922672,0.9577933941193836,2.259175626413194,2.1754709544095685,-1.9126748390242563,0 +1.3362754068776401,1.2071015263717277,-1.5656871389724607,-1.8426637312791994,2.3748365772541424,0 +-0.9878438890073911,-2.2283478993005166,0.09647884946811158,0.3778953737880706,-1.7106384833841974,0 +-0.6995921000121499,-0.2781565433820208,2.2205122182070376,2.3478967053424897,-2.624174237941769,0 +0.38942275712397234,1.1086167968945657,0.005346014015365552,-0.038132509973745954,0.9262293654372233,1 +-0.5483603760983984,0.2246045803547324,1.5757524876203959,2.4386904939590925,-0.5710283439117549,1 +0.2887984783310744,-2.225765327691778,-0.41124288271135157,1.4037684970935498,0.17456258033499705,1 +-1.7194675932886905,3.49563582725577,1.4910663788056069,0.0071082955017790495,0.6199800268409534,0 +-1.641095771570324,0.36700782206395566,1.682218549686627,1.655567895378898,-1.1718092871581167,0 +-1.0837360084944896,-0.012912508425786173,1.9818895366840326,1.9551630985619182,-2.106813149627893,0 +0.5682995631211969,0.6795301163408718,-0.43693465328778436,-0.20465566737967356,1.2816091483084093,1 +1.114623177999174,-0.4797482353480105,-0.19865453171293468,-0.2685196734349298,-0.6883704507870884,0 +-1.0795540158597503,-0.16818533476983055,1.7418841571821706,1.659512082223742,-2.03111672915667,0 +0.6607240777221002,0.6029286424214246,-1.3466800499329494,-1.2601089442740192,2.127870302820532,1 +1.0344020171889092,2.061167561981364,3.2066453961513783,3.0462033259709305,-2.3859289045451915,0 +0.37136626766904346,0.8988589559057312,-0.8181199281274159,-0.610500441496342,2.0073530474906947,1 +3.7116398922112634,3.5889849091355615,0.25090872873830006,-0.594608740422508,1.114082738598523,0 +-0.6197162938109735,0.17086632906781907,1.3916354379240317,0.6515852235281958,-2.140677563898946,0 +-0.3151282280596497,2.3140330376379397,-0.09883021121604015,-0.5162790200935368,2.086682301716092,1 +2.709356609461822,3.227611268129213,1.0996805111753503,0.4119111496268357,0.22195526048371694,0 +0.33384529347991454,0.7616572175642997,-0.09741462686862226,-0.12735033977625054,0.7378804810745756,1 +-0.8423817325854446,-1.0294860412093838,1.6177786440375432,1.9756600339212629,-2.337187834480374,0 +-1.2923255145001433,-0.44032485980351876,1.9596213213023441,1.6639831976262847,-2.7341414527610945,0 +-0.8246931696906928,-0.7298850311993501,0.05459299495176706,0.9840113582527745,0.5016273852505465,1 +-2.248678031706808,0.9247170757928248,3.1083972032029283,2.806035802350913,-2.459750586635631,0 +-3.8332084286955097,-3.1927848638074052,-1.9961075204809295,0.659131258664146,3.454796585664818,1 +-1.2838085755994462,1.5916029568860783,1.801206241737078,1.1038237210006263,-0.9485190719402472,0 +0.1955527031374109,-0.06374443514060057,0.43929618385084285,1.2908962891709552,0.27014857744842047,1 +-0.5150116318388791,-0.7317942871667593,0.20224014659876643,0.3276870790115657,-0.6704129204329644,0 +-0.6183484211323023,0.5910614737199885,1.0104138016738153,1.121662483059466,-0.31541706411392356,1 +-1.1117206206580046,-0.2916144896590832,-0.1403428277478413,0.5031747723035098,0.9612026352836874,1 +0.6806985930859077,-0.37924856391598605,1.292181110236423,3.306184317593251,0.026377881143267112,1 +-0.7664345529320518,0.12088079144993336,0.35907121412088133,1.2259676015461236,0.8930396756946153,1 +-0.3622626006601888,0.035774601592219235,-1.3191452563411281,-1.9347799572605067,1.0954755205955942,0 +0.7438074113115787,0.11163159415571533,2.173713671915017,2.3461213037236375,-2.610714833688412,0 +-0.48135405773256246,1.6487039054926758,0.45031714530886136,-0.01355253185336891,0.7516396829527527,0 +-1.3313860159229876,0.3298098086687592,1.6820742692246928,1.41915847628876,-1.5729955837822076,0 +-3.201839618683057,0.4333381218907446,1.8075318760883068,2.9056901374156774,0.5033912670822451,1 +-1.663699797264159,-0.4778971833613952,-0.8685272153745132,0.07934271843335561,2.186878245479578,1 +-0.04011990176560687,0.3467979463488742,1.7792865631682049,1.8640410783385772,-1.7253397232117011,0 +-1.9700983212922178,1.2523552920846863,1.949547833863725,0.23508371744911494,-2.360153340043535,0 +-1.9138076073277235,0.764395733894055,2.8980653256459368,2.4929620747337333,-2.588259348106957,0 +1.4499153903806032,0.44282039190991374,0.613886141302318,0.31144683650722127,-1.1275968948981936,0 +0.6470644837194861,0.30046407669196873,0.12847765759192872,-0.16581784002844846,-0.39735177152483947,1 +1.6947265626501289,1.7341042770129143,0.5895345458275246,0.09213992065230914,-0.10018132696248938,0 +0.6742874243923125,1.324588920694254,2.251732728997325,2.086523998891481,-1.8388899745731457,0 +1.1406781298884257,0.7317402925321221,0.9674071953350729,1.1902859708937874,-0.5888381734691674,1 +-2.351107596226875,0.3320393505891239,2.5383540909712203,1.879936816264486,-2.7132526067228433,0 +-2.2922099781416563,-0.7762820456223478,0.17260607641965087,2.666820189194512,2.526030997535515,1 +0.9883565057274546,2.0301834171146,2.5645732584655567,2.142848439045399,-1.902687974180868,0 +0.2506410561359709,0.4043234434677987,0.25703804342676134,0.4259526996659099,0.19234538233600929,1 +-2.063942516778731,-3.2362740435065613,-1.0650128942976698,2.238045477366481,2.390451336254662,1 +-2.294813314784948,0.036749828525122874,-0.4527629649810289,0.23352676724312027,2.1170630082453634,1 +-1.2302965864704927,-0.2298466480606438,0.24753480645585169,1.3628471770861816,1.1072737967808348,1 +3.0384155702296543,4.143770640290862,2.467960273050572,1.9479267104159412,-0.46159179072637213,0 +-3.7094732863027984,-3.517098886014722,-1.9352302476009529,1.1939679596310857,3.535558279763149,1 +-0.4999238365897466,-2.1254535517533215,-0.21641095772404007,2.0467239147560097,0.7944964700137268,1 +1.0971001707655608,0.01310228471775654,1.554447681608008,1.6614300046149642,-2.1413912055401263,0 +0.7521860091967676,-2.160039793400006,0.21983101732403232,1.5726801364453347,-1.1970617727144066,0 +1.627547479262255,0.5168697864669995,0.6436096857032132,0.5437260884800762,-0.9263060161592033,0 +0.8451155041244856,0.8960552860738265,1.1229768859120794,1.5928153298949423,-0.2436052520647194,1 +-2.0832120040552278,0.28545220681213024,2.6319185654520636,2.468367033417705,-2.4194740744165673,0 +2.3189355661288777,3.657096913572673,2.532287999620387,2.120658582694909,-0.6675199660742203,0 +2.514804439945528,0.9589232982463185,-0.007801500132298411,-0.34885395731874036,-0.2483678033789718,0 +0.04336559759654013,0.19306473938811186,0.8100341524878176,1.0715254475141178,-0.5254077854088179,0 +1.601712940618461,2.7790084526340104,2.244141054750555,1.9260173812180148,-0.8511609401948771,0 +-1.7845742746585624,2.532392934751994,1.5644199644143153,-0.1512646945552234,-0.6687955168998937,0 +-2.167829291127316,1.092778028707174,0.5781339787714421,1.3682497433502991,1.9844476037061105,1 +-1.6069174440626608,-1.097467148749626,-0.9772245360785796,0.4436156961591704,2.200784328944952,1 +-2.9496481104082717,-1.6069617663602425,-0.3521390305195575,2.0663626166008195,2.473360138385329,1 +-0.5409518022560234,0.3574619516351627,-0.9704789289747868,-0.6606489286947119,2.0698939959991405,1 +0.13160598942733448,0.44437593117930385,1.0959011994986412,1.5256800383009055,-0.4679753514024687,0 +1.0350492547187982,1.9223597492533595,1.294083983295678,0.8979778772937145,-0.4417722309115306,0 +-2.0997817197359843,-1.3401159845072177,-1.878468194513212,-1.820871937773612,1.7257149891786197,1 +0.06841677105976451,0.3447698517857547,0.5021144707531356,0.08012531196973083,-0.7571906372960562,0 +0.6919403446265544,-0.34293463887676756,2.0482636912577834,2.2489084878756227,-2.864196127444556,0 +-1.6565996208480873,-3.8752860887893075,-0.9494448204428232,2.5826560357572994,1.723908612238768,1 +2.563877439284834,0.7863044584852843,0.7058859641405053,0.4356129097848864,-1.2342424504416118,0 +2.2348679265837403,1.3680263865364317,0.6826059929516554,-1.0114090614468005,-2.0801160844972273,1 +-0.7389636816417426,0.3463081423092159,1.051862657745989,0.6638354525128904,-1.1209014274821953,0 +2.324447412224612,2.124082245266079,2.0822674998350155,1.897853448248556,-1.4045492573508662,0 +-1.6577953997060195,3.5431347050524415,0.887952542659506,-1.330970560162208,0.5763947111639605,0 +1.591527790716269,1.6389674367702853,-0.28904557679363535,-0.6074263913873148,1.1125374029502466,1 +0.9768572261504459,1.856721989004823,1.4730920850173237,1.1880251287815673,-0.5850662427303743,0 +-1.8374019353513755,-0.8124716733825097,2.405578214946731,2.451563919134256,-3.094464479303658,0 +-1.7266218841437064,-0.02359013524517739,0.21001239804658728,0.700480313093474,0.8383813812055708,1 +2.1802838750955535,3.470134620610875,0.9561459732068427,0.18220439725336002,0.7234829450256977,0 +-0.61362910569733,1.277457380445354,2.6041062054346176,2.4705938784249146,-1.8508896607627023,0 +-1.1551992864958427,-1.403366410995885,-2.277493613161628,-1.5239670534588141,2.6006806747495803,1 +-0.24768481415092636,-1.0577788044260061,1.6549735985515848,2.5071806681958346,-2.066232394376576,0 +-0.40792379960268454,-2.11614855382016,-1.1867980283321495,0.6400109339000085,1.4812640817047842,1 +-2.6807979253862637,3.814353014575204,2.1489538718525942,-0.1856619125503356,-0.48080369359806185,0 +-2.1564998164785916,1.678196980167825,1.301903311756722,1.583754600957698,1.1210811869866673,1 +-1.0667807573843415,-0.2295574959439297,1.5376945925285677,1.6738964511428893,-1.6060707437602013,0 +0.707515555042219,0.09998639968317125,0.2507098656716912,0.09808059306062167,-0.6123262458677282,0 +0.8938415693004338,2.5214848074697316,1.2823021129819532,-0.12641336366068656,-0.894528237932827,1 +-0.7336436558532282,0.5185386544516029,1.5954444146761273,2.281756873404877,-0.43286474248998164,1 +1.9933451428865494,0.44786633653555363,-0.07911833108975108,-0.29277239663272425,-0.3580705516023711,0 +0.4900666484823464,2.4673661249058867,0.9954950996161172,0.7506916833444428,0.8197752522728632,1 +-0.9467154611854431,1.5903687354854186,1.197831867026652,0.5645744841105107,-0.2530340452560509,0 +-1.9596413035128326,-0.1460075894234103,0.15328380099335082,0.5392273077609883,0.7482246255974008,1 +0.4314459009784185,0.23459775906487124,0.8853239912964086,0.8412438631811001,-1.042697945149408,0 +-0.682381858468813,1.5830405886634114,-0.4626840275989676,-0.3825364806020919,2.470844271870866,1 +-1.8871383808349407,2.310377977420807,1.7028298713560945,-0.23992216781650044,-1.2765598213019305,0 +-0.5286893495491088,0.8307116363396054,0.4096662562418744,0.8083305098283353,0.947211910657629,1 +-1.1086987474820493,1.3403732787172116,1.2708450037571506,0.6790361664064749,-0.4930976132004564,0 +3.7997862565231837,2.8231018862268975,-0.11205294943640298,-0.8358515267222777,0.8954838892407861,0 +1.2439449268414506,1.5504099503577282,0.6516751482305945,0.7993903769664397,0.5011880088559257,1 +-1.3208756567823317,0.5564960244888131,1.813863712236575,1.6260054751303046,-1.4280846947878518,0 +0.5665198988975316,0.7076262142566467,0.7347880628058787,0.5450223673830455,-0.5898556651015592,0 +0.6447791789033059,-1.4895538960441748,1.043697819419559,2.509025163554478,-1.374367048452404,0 +-0.4521339707611136,0.5376527171020264,2.3082890830940848,2.4634897292946816,-1.9663085486423513,0 +0.2890182638735428,0.35289945900642006,0.442788495059652,0.2980419011250631,-0.44505143220716015,1 +0.6464509670047949,2.1225867116871306,-0.6063456200634835,-1.0136030556116213,2.2051870097694124,1 +-1.0043642193681748,-0.36903880276259327,0.4772291711960822,1.8855376757923101,0.9332236243286268,1 +1.4784095104922712,-0.09418865070027349,1.289492743189909,2.75370098018944,-0.5576264030407259,1 +1.549390742441036,1.7199451325613535,-1.9725175483450563,-2.7910439285924404,2.7194507122029585,0 +1.6928486117645272,1.6227137580803586,-0.5475492099075668,-1.0739314699657307,1.1498847530316998,0 +-1.8481648613456465,-1.3680414216592993,-0.2858285550585581,0.8807837107948446,0.8835023260623835,1 +0.823472612805096,-0.4060354773498627,-0.27707799752811246,-0.24382068734272155,-0.30717739220800033,0 +-1.0585819992207868,0.3382306232119072,1.2905920518937077,1.1998223693680459,-0.9868943631319809,0 +-1.7864343328060586,3.043215267226168,1.5400007821494748,-0.33450392032894416,-0.3011015859669931,0 +-1.9895029416000818,2.8699187499026335,4.156990687365398,4.629231263699175,-1.0283862390780407,1 +-2.53345323279635,2.213763631483755,1.0230933022578965,1.1828624489486188,1.991508495570377,1 +3.694382734619962,1.4258867679159382,2.8551143719515646,2.9135921037866885,-3.2432821950263593,1 +-0.3229596889549351,0.41994802499869033,1.315193731827938,1.4319699750443637,-0.9533220376228855,0 +-0.6931680314488304,1.486994519060338,1.331380609980207,0.8156047764102605,-0.4759410493155619,0 +0.9246407877180507,0.013112477663526856,1.0762159780321128,0.9504509778663759,-1.753969419222451,0 +-1.447861253324464,2.536240631419267,0.0924883981082443,-0.6043225320200212,2.147370120503993,1 +-2.3747239989830833,-1.5618373652740922,-1.3289876923270154,-0.18890172598861676,2.1146115977457782,1 +1.509564649318543,-0.22481104481758962,-0.05458564756812251,-0.5609032482918916,-1.2224348551211697,1 +0.33793606237986684,-1.247233208724829,0.798474969612832,0.9685825688647349,-2.1544641452547646,0 +1.924751621477594,0.5780229235918963,0.5940248823663592,0.3181062295247187,-1.0978128016688715,0 +-0.4460891711101792,-1.2951712461677676,-0.5649038941383058,0.22338593428913478,0.4114939359639308,1 +-0.966500140440298,-0.6366280603084185,0.9163111487378174,2.2874505456409953,0.073445795142561,1 +-0.17400538786312814,-2.903320096389185,-0.1333606829552334,2.6655246561308132,0.3936740937102483,1 +0.5111255997619428,2.0182005425810368,0.23740435037226348,0.2128302203626108,1.5337093780153723,0 +-0.26053173704652377,0.5421592810550548,0.7667217505917358,1.1409707911227833,0.10370361678606721,0 +-1.4347634685445718,0.07826675875326439,-0.13708913697587322,-0.3133481960380713,0.533581416407303,1 +-1.5034976734814962,1.2600133231048716,0.6624841303248934,1.0525433707842982,1.385345110412152,1 +0.8038440401342939,-0.626507843215955,0.45114038743550416,1.860911483240411,0.1021506282373521,1 +-0.9515292283701242,0.1881084913634249,-0.014134605829422942,0.6716553705745629,1.2802834845949889,1 +-1.4572742036743271,-0.4183726926174801,0.8219694102053441,2.833545002142179,1.2777053580363342,1 +-0.3926526997626799,1.2007654631451177,1.2153381229573146,0.8759558687199365,-0.5272535031989258,0 +-1.9382844246305173,-2.461563292115569,-1.2740289215648586,0.8721509367474011,2.1069378217632826,1 +-0.21989713609115835,1.3415421889102033,1.4450916681688357,1.5272120985159436,-0.2613779577393751,1 +-1.4141777246533,0.12051094176474453,0.5838403206840815,1.206929589487416,0.5661011228254149,1 +-0.016880670289741828,2.2767030498369234,1.6714025097050955,1.8540292092945618,0.44136896979271867,1 +2.526449395167875,1.068622491894829,1.0035795059678358,0.8585360503267078,-1.165479098302503,0 +-1.433969747690902,0.8212763224947677,2.0251327907864454,1.7015245575969753,-1.5330440079588534,0 +-1.1282998159566955,-1.0725098741935613,-1.3547273924529657,-1.4404886545780546,0.8685921722859722,1 +0.7401986072212519,1.6365521707887303,1.0812963739732364,0.44075984890115316,-0.6378574733028322,1 +-1.387173275186461,-0.39614606721823775,-1.2269272525396557,-0.8043747360904905,2.036634525894616,1 +-0.795996076389942,1.5250704357767093,1.5384306331851958,1.0610917478404776,-0.6146045049495701,0 +0.22263625812384524,-1.3047802076182804,0.9008960533580123,1.4881603410856379,-1.839557211082197,0 +1.5443858592299349,-0.6255963628661819,1.331388255945231,3.277960524066786,-0.632573013481428,1 +3.005767501435023,2.5187206196764462,1.6681122195231597,1.2509193445115503,-0.9864777212480507,0 +-0.36352340972671116,-0.16684293194668987,0.167955596513931,0.5395035180188832,0.15810180803519192,1 +-1.239517116221022,2.0273117274564116,1.144202317662058,0.4372500394976109,0.26772570267282103,0 +-0.13416702072521702,-1.6625681783400759,-0.13785468979670368,1.7714761753858195,0.6499452817249775,1 +-3.029920168534784,-3.865989091891469,-1.5086129953495129,1.3146332915290695,2.0983077881640835,1 +0.366847266002141,0.21570970400004896,-0.12294330117211535,-0.4038148022955381,-0.06551930404120954,0 +0.5263182793485846,1.7378552582708982,0.7733328770619211,0.4588370587988009,0.2717637570064002,0 +-1.006436485878652,0.5958623824296236,1.522909779824074,2.009814501178022,-0.39458277414584364,1 +-0.23785295075569834,0.2745226591539328,0.33446171382180206,0.04969838165357576,-0.3683639269488109,0 +-1.57562170005492,1.1148336279143667,2.0825938332230045,1.4655149355449562,-1.5848387194592077,0 +-1.3686914837891124,0.08413114714779446,2.9594908388160412,3.2079567362267905,-2.8085345914568887,0 +1.9388277077969596,1.8080121199268295,1.4018557474566655,0.43642293344445904,-1.6185581583293556,1 +-0.2708024291048974,-1.213399179913591,1.75252446055278,3.560176669101945,-1.2818059126463028,1 +-2.83751572347727,0.5065305009713786,1.517415225852517,2.426016438851642,0.6012173048097238,1 +-1.4039270307489633,2.3772133357208265,1.5812460746061965,1.5608362608074482,0.8942563671826632,1 +-0.46983663728339675,0.20446418969667213,0.6295149848707967,1.4170033111661324,0.4585698180944332,1 +-1.1411326299750226,-1.0808095794249435,-0.9303719253210597,-0.4315872874466502,0.9884337311114666,1 +-0.3864983747171956,-0.5663222884249788,1.9853803566827273,3.310137582329977,-1.4114100182911677,1 +-1.3653385563090459,1.121187276429203,-0.7646720165500314,-0.620235347764583,2.6777292333399902,1 +0.5348277134206394,1.5729151359933562,0.930634553261211,0.2003244107838632,-0.5470469013841766,1 +1.2402005159207563,0.5404193826508673,1.7798601002124381,1.8210123050195017,-2.009448227415393,0 +-1.484128683651384,1.175047696965426,2.1018729364928053,2.950927145896933,0.035776169970592187,1 +-4.110247628914226,-2.3432613721973667,-1.7986741668910344,-0.05126469120839272,3.1576894524271553,1 +1.7239057056073237,1.7707256742112425,0.14308125065079658,-0.21159368926852284,0.6308496017019622,0 +-0.005717840815205877,0.1663183298899047,0.9489920933045253,1.6676758429566478,-0.20250952103656872,0 +-0.13540959899611626,0.09491482743305957,0.007346254692475712,-0.18485663047191414,-0.08009521221481708,1 +-0.06901089785943548,-1.7301495402931586,-0.2611341086561766,1.371908794248256,0.40711228185221127,1 +-3.0652002456925587,-3.0243078532289234,-0.9707380976453992,2.1923744344681415,2.669187488028802,1 +0.47733585598069317,1.2117897339057686,1.404160816261035,1.1334296653315266,-0.9637575358569638,0 +-0.9080325270538138,1.784936004622983,0.7807643141990306,0.8063399236328678,1.1655004270519547,1 +0.8296511948632872,1.013896243209183,0.25928354286151145,-0.397324083527707,-0.30268811666561235,1 +4.149420612171177,3.724728747712782,2.2135084047666047,1.5782832038783856,-1.0694551081379098,0 +0.8792426337044599,0.666226960430528,0.2679966816611035,-0.027296512565038844,-0.2806108575003712,0 +-0.8328459519825343,0.25349701430025795,1.3443952031125588,1.589017904677461,-0.8441197646879243,0 +-2.276993236581929,3.880310067429613,2.021651267086897,0.12570973858639822,0.08889954728029092,0 +-0.37285755842794366,1.0369447784594232,0.22645236997345336,-0.6425247340471605,-0.06985725800170972,0 +1.6616286481781595,0.49289547788000054,1.0144888580934515,0.9224387719484155,-1.4075002055814465,0 +-1.353009858444575,-0.22968884698665737,-0.5067953535250777,-0.4871118206581573,0.8661589063174681,1 +-2.144018488267287,-2.204117341802065,-1.0208419892602494,1.23357166042503,2.2430342381306883,1 +0.6673397714009341,-2.3282291245851643,-0.2174283017642718,0.17594508510677576,-1.858131437228015,0 +0.40411269970304664,-0.04680022081142288,1.0830303280247255,1.032813003451691,-1.5645698268524086,0 +-1.7938962573228752,3.5708911598147495,2.94871873694966,2.745099082973759,0.3446045158940314,1 +-1.7207724052183861,-2.368476485800663,-1.8032892858692982,-0.39838716469704827,1.959346429669739,1 +-1.7565748774173446,-0.4720473011661368,3.1758802611569665,3.7593278419092133,-3.1321387853689777,0 +0.17188927552942923,3.2868223972253383,0.3476785525345107,-0.5264724390427524,1.8485013113517872,1 +-0.0026480153287329333,1.2714052342773883,0.2689148759104517,0.27650568643253226,0.9542753771733583,1 +-0.19396964661345129,0.3168745896549272,0.40505654736197344,0.7903348959346863,0.3107623561309576,1 +-1.0001896419257839,2.236371836225418,0.5631113375337815,0.05560858335760965,1.3285327251980732,1 +-0.17693960492790461,0.4134706102666038,0.6163002737204041,0.4758005374396731,-0.4359197696603946,0 +-0.5160561996653825,2.861890346522963,0.7842422319410532,-0.4188783331753858,0.7562134462293799,1 +1.5187962406718318,2.3028500427631395,1.3100222415026537,0.748354312045072,-0.4244844325378191,0 +0.7195649770877928,3.499969575826726,2.3969252103215752,1.642469893865924,-0.500228403899468,1 +1.1719355838562509,1.7624036656846473,2.1405424141024536,1.7930458818060795,-1.631479405066993,0 +-1.537155653914023,-1.1430079975241618,2.5640519702804325,2.9285498817744973,-3.3698980718380476,0 +-1.3303668461572835,-1.6936655415019657,0.1773437990241673,2.992470708418673,1.6313886235603063,1 +-0.11345100289142196,1.2757756482670777,0.47708061662551604,0.43195930792105586,0.6827564553072836,1 +-0.7629179949483428,-0.27617618588886494,1.6418570359774303,2.065451141500074,-1.56585039423056,0 +0.27381437870660386,-0.9383735463170196,0.6993792590251338,2.169402481165384,-0.2704310280433495,1 +-0.9791622407826074,0.270416996696901,2.0304469526919258,1.8343277353937997,-2.1039083332238695,0 +-0.49317933220354093,-0.469908183137298,-2.311557095863225,-1.8812150755024732,3.000144029002528,0 +-1.1583689891285105,2.484658669196411,1.3413636594175335,0.4204057953671425,0.2217444390214922,0 +-1.328374813979823,2.054148338194013,0.6209088244945542,0.5709631385616609,1.6890319907435074,1 +-1.4433267867550246,0.5389333818433084,0.05596400460981754,0.8055529275293867,1.7813067780769112,1 +1.1255558323243133,0.8713903448230291,2.5295266785617008,2.5832227537339847,-2.543809498996042,0 +-1.9077120184865701,2.012656231215293,1.8963976913128644,0.46297139305479584,-1.244343659147184,0 +1.7140231039399492,0.5071868918665747,1.2425120495144975,1.9481669312034033,-0.8116313158407906,1 +0.10765372117260164,0.008477535250883017,0.4969757206503288,0.9564618719748301,-0.13044379468479173,0 +0.43623422741147366,-0.993439702588401,0.36420887663004015,1.012663057589386,-0.874247679269517,0 +-0.38776382127101605,-0.08182069451372143,1.0540803991215468,0.7801751938791066,-1.5449116845723283,0 +-2.7982373866090287,1.2723319992175555,1.8145640403062162,2.4480966048395794,0.6887414382239595,1 +-1.0098201002277636,-1.1315710190249835,0.06808451769622126,1.5459025073677108,0.7484647736159963,1 +-1.8012581917965789,-1.2400219157024002,-0.21713129920110785,1.5226413466107764,1.5431235820133866,1 +-1.1843282550932224,1.177110446385258,2.189625540017001,2.6122227628217933,-0.6398582723060069,0 +-3.0425313386137356,3.4612277851324533,2.9673163602028954,0.64649521053878,-1.7008058713052556,0 +-0.9501769856512048,2.6863447252824826,1.4481122133444377,0.16095680400245038,-0.17996733110846885,1 +0.0070065087021962125,0.30716593329307795,1.6659192709592494,1.5951034761800502,-1.8133120082616032,0 +1.2822329830386654,0.09144681253352793,0.3460389984813706,0.18807937256998708,-0.9363283648253663,0 +-0.5716175052931314,1.5781398662006467,3.0564030733390273,3.026876547086267,-2.003161067061893,0 +-0.9026113290819358,-0.5224259307966588,2.4894046721277925,2.590331889504599,-3.1594808993814048,0 +1.5829937155239604,-1.2071240414216056,1.6532662227900978,2.1237543707526227,-3.2483026275832207,0 +-2.3680471278885697,-2.15684179236145,-0.7741246549537151,1.500486331966433,2.0855669633490037,1 +-1.6838723397919555,-0.08734707306304179,-0.4801719039874643,-0.3303521632621904,1.2305254383673982,1 +0.801379885979252,-2.6351654150201567,-0.19510740518740066,2.3905288601758707,0.17609290414542078,1 +-2.532040105668484,-0.5900008465689135,-1.6179032711574552,-1.935003527417177,1.8907936297604286,1 +0.1456884132478502,2.0820787964025005,-0.3505622039316928,-1.0233533274210358,1.7267885386514572,0 +-1.471730687738439,0.7042799057666588,-0.28328931932744356,0.1222656291738562,1.993327833140542,1 +-0.21877728727881007,0.3273165792722388,1.6698874657906624,1.621856038253389,-1.6971449796815496,0 +0.19833479844033852,-0.05105565055445194,-2.1345001702733297,-2.126707512408098,2.505803869865361,1 +-0.3384918033439622,-3.1185950322221307,-1.219869903917575,0.9442271183634823,0.8650616053327775,1 +-2.216001962437797,0.3425060939229678,-1.0097165848325518,-1.6164461912252541,1.6555064522604797,1 +-1.645547608357818,0.18892637083347075,2.126344615203061,1.9906651898105288,-2.0129830616892694,0 +0.3198807491168083,0.271214432333138,-0.5313207139639615,-1.5317303164853517,-0.2862159027618647,1 +-1.3305257118168767,1.0000474937160033,1.8638032194058574,1.008858882677166,-1.7760290437478667,0 +-0.14186246014788073,1.9380524371343952,-0.1676419749266197,-1.2526898465888396,1.0007870669823262,0 +-0.7777906816441998,1.5216542476681885,1.1367184761832103,0.06482971498336942,-0.7867490307177385,0 +-1.3711113753391018,1.7012368730519483,2.9944207726069605,3.611467564510355,-0.8236150646458953,1 +-3.5698645750027986,1.281549646194884,0.801958064589998,1.2375772796196842,1.9793921669505454,1 +-1.705554974115564,0.14213596459546918,1.3680483221672595,2.025481914233727,-0.2373760629193904,1 +-0.41122809910499314,0.7656073423046565,2.281126834239826,4.085357506549949,0.09772823370246442,0 +1.776950520686271,1.1707804561648383,-0.35934093414156837,-0.9539216447576471,0.3629776349908491,0 +-0.6797821659099381,2.9728826240702846,0.27980648073495445,-0.11397563857829862,2.43164421738636,1 +0.9792612862201231,2.3096035037145723,-0.6623856874455953,-1.1767534758755054,2.2317142101811447,1 +-0.5194548731507658,-1.8611682968692314,1.0344207091386592,4.218275539692498,0.5475366750097552,1 +-2.547647060850711,-1.4749665423136045,-0.8816750506890066,-0.0958061576013649,1.321626529009071,1 +-1.1494131177027485,0.05176240585600911,0.5166794385294907,0.9909250748592698,0.3267376207357362,1 +-1.6090392983115651,0.9445539263957552,1.776702778069462,1.9933617647175557,-0.45143251943709495,1 +1.9218431160822096,1.2803774939366295,1.4005179822140659,1.0883824546119931,-1.4206909464193749,1 +-1.18971013985211,1.494949382889423,1.4631918883697768,0.5517447149996417,-0.8986128261731343,0 +-1.5992718521471114,1.4518238386046303,1.6768510240317278,0.9806251193521486,-0.8291957705924676,0 +-1.5038643270244751,1.236400746630434,1.7836141643376533,0.4544378451425658,-1.9049659642393242,0 +-1.2065399080253272,1.6027538155903573,1.0338117032374574,-0.09215526864833201,-0.4951162750254262,0 +-1.18419496610267,1.0332346012095546,1.0969932817834698,1.9412946918507852,1.0188329613847114,1 +0.33204437903351525,0.5488852674267048,0.5797586624972211,1.3117446030022313,0.5346678691489415,0 +2.5012255989618417,0.508314601057178,2.186470226717562,2.344546201369118,-2.8340036227623786,0 +0.46119749083218653,2.586734150636205,2.8072378030952096,2.6094266849386427,-1.2186632065343408,0 +0.15613018524727362,1.2140740460768482,-0.4890824830250456,-0.8372870043881373,1.3803351063559552,1 +0.24789232426352537,1.7932907754121028,-2.0335588044533255,-2.8515779784766613,3.3048935105273314,1 +2.6434253734868594,1.980239090321485,-0.7290186451547664,-1.2927668285242668,1.3705709464840177,0 +-1.5100594086482027,1.5047071659604536,1.7465651350606572,1.094275146527267,-0.8431321559064526,0 +-0.10951270911021715,-0.03825472735575608,1.0300132019037473,1.5030016545346672,-0.7425850301829096,0 +-0.9864635572748018,1.7791207375463336,0.20430972510145423,0.12913464649312778,1.7812321756445741,1 +-1.7188054836648354,2.1269667901634026,2.179443862242762,2.5361067603036247,0.43143923343390034,1 +0.9213887048698323,0.4465927587684355,0.32392015536577046,0.09516809636412182,-0.5101450475171183,0 +2.769601769634144,0.25916500307055035,0.9485429107607269,1.004721440840574,-1.7694340236915853,0 +1.8591468380285128,1.8374819247408811,0.9557563786305253,-0.3378606825209318,-1.3771915083804727,1 +1.8448906938165135,0.7661158425755981,1.3340342869675363,1.197776640723994,-1.6352472234745952,0 +2.6650337513580835,1.3906442223205389,0.43491726714576057,0.022992808719987434,-0.48641258096136697,0 +0.30580303266224806,-1.2909577874562497,0.8462558662407612,1.560450511198935,-1.6468799862779573,0 +-1.6588095901314563,-0.7850500219958799,-1.2557530117488673,-1.2267507307486896,1.3398326939895422,1 +-1.6345041678711967,3.7197854847320793,1.0076916637382267,0.12691358336933145,2.0725878172571806,1 +-0.4559165174615557,3.4709742535130954,2.556977707536956,1.9739990295687497,-0.14222390844416632,1 +-0.8071537472670868,2.374192210362553,1.334355834710894,0.36670225194067285,-0.049639837090545424,0 +-0.058383333399446236,0.5399739059782869,1.1375084054926567,1.6389524677538017,-0.2804674145368551,1 +-0.3606607538960115,-0.4527521623868842,1.6396495464047507,2.140515534208917,-1.7899879916979002,0 +-1.6911275743002423,0.3559282749924805,2.411576223272971,1.9782777314026871,-2.507470447506396,0 +-0.8305961281561369,-0.3744856780445509,0.39856041310159307,1.2707529204521435,0.3754567960985222,1 +2.2687351329289687,2.9830955012692995,2.56097721997783,2.207801091707426,-1.2970819137824918,0 +-1.267509822926727,0.23678705563230817,0.9096634785153505,1.5149322583130218,0.21467468635548947,1 +-1.7353818017447797,2.547248621361577,1.6823822855515067,0.29899906803575427,-0.4489748013289787,1 +0.9334768437079146,-0.16851626636402983,-0.24578963088546169,0.4354525384328032,0.5692703128165573,0 +-1.4938324116807502,-0.15268610379923309,1.9307549161137634,1.6601758136830027,-2.3152552177458134,0 +-0.4502228007340068,-1.2716996565842793,1.3079613352644621,1.512240346924726,-2.501135566776727,0 +1.324980961125342,-0.3401760256146382,0.8836305073645911,1.7043861660027737,-0.9641224692528512,1 +-0.7690063883819765,-0.07973281674275512,0.28288231158829635,0.8891136972674495,0.49908782382996386,1 +0.16922253367772466,1.5810197703323663,2.3019547287415922,2.184329532252888,-1.4215444310975924,0 +1.5917829538733783,2.8858079265047962,1.6253992315567352,1.027466688378091,-0.2909022545011075,0 +0.8523387819187136,-1.9320318201107458,0.433837586427528,3.200783971330458,0.29298608112612257,1 +3.327346855388015,0.8951432849423449,1.6088840668496356,1.4983745743433265,-2.3115494378658754,0 +1.7559324372435539,0.9897838443788548,1.6111419431509204,1.4598757906039657,-1.7372062358021596,0 +-1.9735935411217058,0.8749862984359823,0.6368795555971938,1.0547161003369563,1.2190385664452061,1 +-2.701054660433766,5.90959121657059,1.9476797271492183,-1.3881667994873026,0.7702608282414367,0 +-0.9149693927528287,2.688451944110236,3.1941438773235964,3.041802086972155,-1.078989477916013,1 +-0.06018688702356534,1.2908047297399743,0.4782912825262234,-0.04693557786941449,0.14975977877073077,0 +-1.8555944689443753,3.60961218727843,2.2715542927827657,2.0126809111764254,1.1728475886609593,1 +-0.42801957388106504,-0.06948587306147358,1.7706080529346875,1.911260563885678,-1.9403188502360262,0 +-2.1596784213490743,1.2788453783130547,2.595165356307195,1.5621503652583373,-2.310498862214711,0 +0.9414855632001368,0.9227114717473404,1.042264662311528,0.744181803804657,-0.9959987974590196,0 +1.6488479139843257,0.9428412154598211,2.134451713722145,2.211941826930293,-2.137521220187744,0 +-1.2000326867057887,-2.1077527197382775,-0.5328093517240292,1.0373935968872638,0.6716165661072553,1 +-0.8084390039700073,-1.4165174105694187,1.477978242515306,1.892622537003421,-2.502761870717478,0 +0.15187062556406405,1.9064125314524154,2.371472348485823,2.066468618167821,-1.3810601298145497,0 +-1.7919595850441108,1.103751066073377,2.4063460159007777,1.6896261408035547,-2.0297291696701576,0 +-1.9175753084317257,-1.23788015287841,-0.46782512953916533,0.5276167356473772,1.0717211672524734,0 +-1.488633857368394,-0.054646451880202895,0.04944845900235584,1.0186175727199696,1.4512868745224219,1 +0.9630443137692635,-1.6655367739939968,-0.6051843247798073,0.07372315892581582,-0.5035609691525107,1 +-1.0481334991429372,0.9949073653800542,1.5551344233455768,1.2694636619927464,-0.8708579485147733,0 +0.24698901691181208,3.4943141281505934,2.9529559016225866,2.3476867451124566,-0.8642658577880722,1 +0.2161563918790428,2.232362506731066,0.42817327737166155,-0.08800682845512564,1.072250189886585,1 +-1.3444934432122482,-2.3891419098954243,-1.2638915660093333,0.6026356912405605,1.6599656064849952,1 +0.024415716168932455,-0.5688909420701944,1.136592620200569,1.4477804732226938,-1.628175495545304,0 +-0.8760802559766744,-0.8858958942335887,0.7415445235893364,3.017439355329094,1.0037023809512002,1 +1.4970206461238103,-0.17924232453956623,1.7480736737991331,1.9198234729668364,-2.6342334414992377,0 +-2.9025732974130434,-2.1337508124847218,-2.1713698734025444,-1.6794373801209845,2.036596142332521,1 +3.276812503962505,3.84545310187302,1.4819977972932543,0.8001318903547774,0.1890681655749935,0 +0.6766897858461342,1.404571878480432,1.729646874488859,1.5059304636317252,-1.1843546640647629,0 +-0.7135573328012883,0.5311222557399757,0.9462466516077931,1.452088641968583,0.16958854211931151,1 +-1.3972589585427044,1.472933388058028,-0.35682144099468693,0.08702550420154453,2.8711042222151395,1 +0.6906815447632689,0.7094265813605418,-0.9940578954910266,-1.6290591370732659,0.9979612167252345,0 +-1.7793054748284316,1.6704589375456038,-0.45349722795737446,-0.6935535354617093,2.562555157686379,1 +-1.1615026975667586,2.0879940663508916,1.4943596157311,0.4673708277793055,-0.47902276255846843,0 +2.3621090732510632,2.9345721896907975,2.1747106805848198,1.7372013368390187,-0.996708123507243,1 +-1.773035253504721,-0.3358926167437134,-0.4630227180499953,-0.2896108939917339,1.016261529852953,1 +0.21572857739644924,2.1849131351179976,1.080607699196841,0.5072129426370945,0.16253912483845467,0 +-0.5891073325387702,-0.5782993668603442,0.5129727325116784,2.182383934423773,0.8279108701518112,1 +-0.2517475870977097,-1.690790551283349,0.43106230786184474,0.6743782165073311,-1.8705545476856598,1 +0.8902195738983008,1.29733536397171,-0.7571090998042564,-1.2212095448940619,1.4182748479696587,1 +-1.1415177317497691,-1.4521454254737503,0.6297561398641274,2.0888949897271205,-0.23736735681445098,1 +-2.7375429020405004,-1.2065243656019469,-0.13881985429718968,1.483703793016306,1.665717075065149,1 +-0.11182890040764404,2.063950330835677,2.8006501966041695,2.647544230590724,-1.4931866959602722,0 +-0.5582241182706771,-0.6012587412974444,1.9920417278712246,2.91430923374521,-1.8402247431968004,0 +0.5472307271891212,0.5897747826491881,-1.002535729832236,-1.479547706911628,1.1105110842520363,0 +-1.5764170017068844,1.5642035901610654,0.18269992603839527,0.1892190801545801,1.880132773231884,1 +-0.5308057222213234,-1.164979639692924,0.05428854092936475,1.400273435308196,0.42601376650224676,1 +-0.6907888013513492,1.1533964105469106,0.8206580601626711,-0.24810467652423096,-0.7944373699465893,0 +0.11330849715657765,-0.27108234923781116,0.6243441834275627,0.633700775598711,-1.0643116796696994,0 +-2.0969545097883238,2.9995467569782326,2.414794873631009,0.8315396848223107,-0.9916690303296287,0 +3.160678770215427,-0.3479078561204141,-0.08015416312560203,-0.17058829255971242,-1.4103756103197547,0 +0.3557135510383428,-0.1021811014568863,1.538804294177217,1.8323536772689688,-1.7836382873168624,0 +1.523679083144863,1.7032106974118095,1.9455119996357726,1.7826254610201802,-1.366578072510832,1 +-0.443694302053508,0.3199184681264716,2.2318727346012714,2.3141032710516676,-2.174140228339026,0 +0.2267364292742684,1.1204893029538288,0.5631513340730683,0.34899692435067053,0.1213920464084447,1 +-0.7138972900348539,-0.5188473758238259,1.6653338683950802,1.9963663403407619,-1.956243858790078,0 +-0.41008006659012,-0.22067762119657863,-0.19321914934776818,0.4849390561182194,0.8998738142180547,1 +2.3528239387054635,1.447600522852698,1.4958458577447777,1.1658583432655234,-1.5340788255919944,0 +-0.9698667180222789,2.519745420244412,0.883880433755537,0.7587356967071283,1.6305182557829405,1 +-0.8408049180962189,1.3073445737517169,1.037952645082397,0.3097066135825779,-0.48100823703774787,0 +-0.8754693856176908,2.6144944385762487,1.5782160705354282,1.7486608359493474,1.1687295719753499,1 +2.1083816800219592,2.614232609896483,3.1398907182640823,3.1124784369710365,-1.9635145932819413,0 +0.7236668628225562,0.1683242179854031,2.247849000252773,2.477760657192853,-2.5746253914494406,0 +-1.8109867070276398,-1.891841329052113,-1.0640913632434545,0.3195167719960087,1.5383667183523715,1 +-1.4296093594225097,-1.7531032446502963,0.37878855241423315,0.39386110364301596,-1.7252206389205746,0 +-0.456303082176031,-0.8915445608665182,1.2936445067970495,2.291116953186697,-1.226988656490396,0 +-0.66278643585668,0.6162581090701533,-0.41110134218053124,0.0386615153977421,1.8390007432939135,1 +0.7487735326691423,-0.95651138909732,-1.2779626285674044,-0.05467088294786904,1.702754091618667,1 +-0.5236373291767441,0.7542031434504037,-0.07430606922740796,0.02230951134950876,1.1291004878836166,1 +-3.288836655316611,-0.581264002887689,-0.6467283348076541,0.48363778029833937,2.557657695483859,1 +-1.570831566728609,-1.7342633401406062,-0.39366093395374935,1.506131601998484,1.3628784660165416,1 +-1.7580967114262767,-0.671632653808919,0.699097378318631,2.8393999010237976,1.4170316738424087,1 +-0.2774936151877422,3.1954625795441616,0.8171782375525725,0.17191412238983284,1.5845498759804093,0 +-1.1322226060679792,0.5506358768789636,-0.5894208340290353,-0.19238780464665006,2.091077722738583,1 +-0.5966721610670477,1.7330820255289945,1.7347324051017243,1.909224209527983,0.0043833089628941835,1 +0.5307883415429602,-0.6785132068020279,0.10520625303170589,0.72185228667986,-0.3080095486977944,0 +-0.38788224291479656,0.31139992454797616,0.8671257597772843,0.7527755487770244,-0.7459609166355752,0 +0.06800799056882667,0.4514059624425565,2.0716078647052183,2.097063519806999,-2.080169554557494,0 +0.98511330861818,-1.5219536295651173,0.6483100214791248,2.905291898718803,-0.16482507693906068,1 +3.1560749763335956,3.126055185386031,1.8837236250112832,1.3791257417722829,-0.7884941256972091,0 +-1.0175436476711797,1.6804446353937283,1.3779747352861762,0.5930051402724767,-0.5267429234359844,0 +1.4444090737397222,0.04829045586017933,1.8050646979722302,3.5113456159708054,-0.7684347740987922,1 +-1.4170234162668867,-1.6921268815511263,-1.4085582428228889,-0.48725582650640265,1.5192988686979367,1 +-2.686693216551752,-2.678464935177344,-1.4861200605947151,0.648800566802079,2.38799282314348,1 +-0.11602321598477128,3.858555785702089,1.715166513580368,-0.6198092205992156,-0.7658084202591131,1 +2.7004629848254025,0.8565439076770457,2.5497831912264033,2.679762800970463,-3.027796143696498,0 +-2.257222392597491,-0.2730152242941035,1.554985203118055,2.735052219313946,-0.12194112971279591,1 +0.8766567725406371,-0.9074525321093523,-0.7615188272898079,0.23089505135428678,0.8220968556120742,1 +2.0068324766898495,0.5536897626235654,1.8189768655657037,1.8060815107932464,-2.3607493865070532,0 +-1.2199328094602708,0.4566426580298767,-0.9148930858857474,-0.7143674280298509,2.2086072993981625,1 +-1.0545491306018464,-1.1667150035352236,0.9956658649604131,1.3299078148463799,-1.6674723951878199,0 +-0.1395757973777425,2.643555230869384,-0.2331323036822578,-1.8290986863598513,1.224810058312603,0 +1.2453414026136969,-0.46462281614243683,-0.7763111001506413,-0.24726982076077375,0.6501575297356373,1 +-3.640081404011381,-0.11990447539307736,0.5178899796948309,1.3925893754311534,1.4296526203456632,1 +-1.3493959574253283,0.9858507105031514,1.7537444521891752,0.8426300587819658,-1.7109561571026615,0 +-0.19083777976506122,0.6065206600018778,0.9049000746692073,0.8475501532134864,-0.49973195884690724,0 +-1.5632796454469933,0.7416870801582529,1.5981424841314364,0.8506444019953047,-1.513156443324729,0 +-2.153588058004878,0.7796355755353925,1.0804664822193728,1.652186668096935,0.8098701445318062,1 +0.9959353011812602,1.449443322428431,0.927673990140043,1.2814710464711445,0.3719968942351898,1 +-1.4182143304775856,1.0469901609519427,2.5191864522352105,2.5427278315498008,-1.5350308127499308,0 +-0.876735111102997,-2.110427240389935,-0.602463587409089,1.3378372176229663,1.053385622746398,1 +-0.6529429564127256,-1.1608948988120233,0.9258363336804656,1.131039147894836,-1.8529115526360482,0 +1.3990084229251694,1.5342844167942502,2.1015267742118295,2.389986822018118,-1.1881806875867937,1 +0.6808124881003792,1.0414607269251672,0.2962953646541774,0.528439971746496,0.7082574972726804,1 +3.4554202561303633,1.6959212064291327,-0.7959520694438482,-1.57219782531345,0.661175761561658,1 +3.8665136782701204,-2.044842175492316,1.3452472104021156,4.686602434062745,-1.315173426145115,1 +1.5494152436192974,-0.11365639202698308,0.5913889829009933,0.5465279346165399,-1.4075753654175231,0 +-0.6030418957453034,0.19944836600944682,1.156544862701827,1.161503766555847,-1.009181303825639,0 +-0.76681573788926,1.5866540820454882,1.0593898672808926,0.42683791514480096,-0.14672491419122735,0 +-0.3910275518201166,0.19659793844158002,1.086817972229217,1.3815441255620944,-0.6786334497093859,0 +-0.9265849202018711,0.7980436920559595,2.1604304990274366,3.526166837959819,-0.031908363214988356,1 +3.1127131657114493,4.277257032078616,2.93207409161902,2.399106326355807,-0.935526208049536,0 +0.5251160974614526,2.455560570539161,1.8860235015645155,1.5074314907937916,-0.44211702886658866,0 +-2.8938234787344186,1.1847271095538785,2.4596198177862814,3.0239814767317106,-0.23342883165350115,1 +-0.036603873707149126,-0.08978642912210888,0.5868494442657264,0.952035694424438,-0.3945033655008592,0 +0.0908531621410702,2.725352131469817,1.4443506367230021,0.9309329596276316,0.3666049688839732,0 +-2.040782595342465,-1.2618020112958603,1.730672395492789,5.722230540968317,1.6951819509458008,1 +-1.7826215279889261,2.411124744932276,1.6644222323436262,0.24283444944383492,-0.5896503709066165,0 +-1.0847774876769953,-0.4981208393778691,0.6258792392720406,1.9793400645040389,0.5882961348450608,1 +-0.5248498792555738,-3.4254630303462603,-1.7781575761409902,0.411094759636623,1.3316154541210494,1 +2.6068427658039663,2.018125509836623,1.3512424122575768,1.0373640488612226,-0.8524973684407908,0 +-1.7836452834718393,2.663678230605787,1.6562758374397533,1.5233837446424625,1.0929990063291457,1 +-3.4897534428022814,0.6808892475210913,1.7952948668635655,2.7275243174189505,0.6803821972080282,1 +-1.3803238098579627,1.9574131869020426,2.013724069411576,0.8353446189270175,-1.3394959188993578,0 +-1.1939037576484306,0.7937519701457905,0.4234457433310308,0.7643150189376775,1.0527790371393708,1 +-1.3093216465162638,-1.7732816494193844,0.08282481558493404,2.8878431969154175,1.6491824072614776,0 +-3.155228856947394,1.9764146706475758,1.1225602319490682,0.8160525335239222,1.3268213574617478,1 +-0.511780820057914,0.20464328232417683,0.10940965251305812,0.6552620368281307,0.843890208535902,1 +-1.5235159845879518,-0.986812378380802,-0.6675434683658615,0.38794739418293966,1.5020079202734378,1 +-0.13039556802590058,1.0647890814253471,-1.5126359344858669,-1.3884417967681308,3.1009623936536013,1 +-3.3860883509310513,1.093529664186957,1.5067743442413417,2.2024906746957145,1.152245915537767,1 +-1.858085434430539,0.31162720992352744,2.744515507535854,2.5915583067675727,-2.5950268654947384,0 +-0.028966541220662334,-0.66246243969678,-1.2206923860911534,-0.7659900190170226,1.3418387635169227,1 +-0.5255570855869238,-1.2759556559746237,1.6637387449055816,1.948026997774746,-2.827877342868727,0 +-1.4940387174298784,-0.07812800183101687,-0.7869899489637688,-0.15878544179582277,2.078780937957426,1 +-0.8708683955243985,-0.5148833170873108,2.425100453500221,2.71109037101316,-2.87999184818086,0 +1.3820967540181592,-0.0384000208125459,0.9404258842605887,1.513196123173913,-1.023448682742154,1 +-1.8400039947331956,3.41720808117424,2.8841704301951063,2.534845370215021,0.12463772508150395,1 +-0.7679815270160875,1.374659791583743,0.6441817141129723,0.5609736576504182,0.7547833721538099,1 +1.5502728560130177,3.5151776227337694,-0.9334955905080059,-1.927835145711358,3.05221786080825,0 +-1.0133976684445691,-1.855639883286227,-1.2747199766364345,0.18755074943697325,1.6517711049969923,1 +-1.239807076172337,-0.22556731119382278,2.193866273719335,2.1271557946394806,-2.571347534830145,0 +-1.0497735623524769,0.247319707162016,1.8318721304522931,1.8986609604660583,-1.570580245815047,0 +-0.6339883801771671,-1.9146089350225837,0.1356181384856072,0.524507799177067,-1.4444374016826966,0 +-1.0912524439034992,0.2707191822792726,0.14815714780438471,-0.03176829859350527,0.25778176049686197,0 +0.13316114372665955,1.7030853775952708,2.7084748047967007,2.6938057970569473,-1.6717276968595385,0 +-0.891771641065202,0.983554766350477,1.141386508387345,1.0672193660649836,-0.19489851689757443,0 +-0.4897488164627559,0.9382719473258937,0.9092157469149597,0.20771416765288964,-0.7815656123062925,0 +0.11602283678192371,0.03852026260813224,-1.0618618882344006,-1.505529142307966,0.811974261123954,0 +0.17158098962972723,1.1950474927587136,1.1133605011396646,0.9987817277995561,-0.34976143618156774,0 +-1.0372066603137116,2.053832240738945,1.1966146651218597,-0.6463644879956312,-1.0888018943553013,0 +-2.341385680295058,-2.94516533941347,-0.6555431281411428,2.8588393728396557,2.5064145398475928,1 +0.4271245540015699,1.862571998154856,2.0063998559924503,1.8059528039618238,-0.954954251164607,0 +0.7083984253481592,1.723196084481783,1.579087277008166,1.4641590309694257,-0.57134963701187,1 +1.1071903088718493,1.3254449036234064,2.0068424975336323,1.8712951164992195,-1.650572969176435,0 +-1.5950156404697586,0.1518178979248107,2.032831028401421,2.0467381593890392,-1.7878393921093156,0 +-1.153290167760378,0.8996108601316863,2.2389567499614937,2.018558520027241,-1.696965512875964,0 +0.6301771672888119,0.5036283089102747,0.37958813535872377,0.24612510632011986,-0.3185480448596718,1 +-1.1856054400060805,-0.19328275201263567,-1.752871875935768,-1.127766317156698,3.0397551824193445,1 +-0.2647356337079323,0.45934449712542824,0.7237168699687244,0.8428415441660841,-0.2061694839032533,1 +-1.8675738663873889,1.4602879908382103,0.41882358349927695,0.19834591387549128,1.3343856624852122,1 +-1.5831967483394171,-0.21754303143273734,2.2150056967825438,2.1409498631829798,-2.482101949340053,0 +-3.6831488583181065,-0.8419704113106123,0.49093972074233183,2.05780331485789,1.5155483865760835,1 +0.4428756247269867,1.642005742950141,0.27362896621613264,0.4796798257688849,1.3890974763219601,1 +-0.28247435788970854,1.3790458798442204,1.353648274181296,1.8261274356087134,0.33911155268972815,1 +-1.6573393344967426,-2.3720569139947703,-0.7465320349367637,1.157968565362729,1.1900459001620487,1 +2.174843149496501,1.2456309040933018,1.4320801023695466,1.8305273424236708,-0.7965325878682232,1 +0.7406299668606107,3.932054018809741,2.9213796216874437,2.278736120820226,-0.5935350568181474,1 +-0.12019434429737885,-0.5097526528185907,0.7732558229197607,3.178200213703147,1.2304596762652156,1 +1.7337767175294372,2.3290760578120073,1.8801083886930934,1.3634508773440597,-1.119142689366958,0 +-2.444520399945463,2.5132380973507087,2.526128098224236,0.2700893174138893,-2.239849022074524,1 +-4.70519490090936,3.2492453248459365,4.2564335763963586,4.628980947114554,0.031208998567317625,1 +-1.919551280266035,-1.6453047505527714,-0.6631506675782501,0.7495000373525909,1.3627138110337107,1 +-1.1468513927273265,-0.2133287920030208,-0.29695435707432677,-0.14941715117483856,0.6971293391478749,1 +1.1367728545851543,0.9175568488168211,1.7466491698199405,1.7278272296270771,-1.6220673407996582,0 +1.673031335122776,1.9820458510016563,1.769115188147199,1.461129373959288,-1.080859469358661,0 +0.1807265107066609,0.02793496612491364,-1.1491126072671978,-0.8761919543569177,1.6756834906086073,1 +0.1321620563744188,0.33743983555836266,1.2732231306881592,1.2632081287175376,-1.2769078604214439,0 +1.8798256406692155,2.6530863812138006,3.2422277193376106,3.015987008182914,-2.192227951175319,0 +3.5921605710401807,0.8313301670609557,1.8514055498211393,1.9504786307942443,-2.530654061816821,0 +0.749024358376073,1.0735761130305639,0.5438720047423038,0.19580859275836593,-0.2246805197546642,0 +-0.45712891775856734,0.45288526721168276,-0.24662624451086435,-0.6747731182570087,0.43796261944830317,1 +-2.71194067566681,0.5788336594263372,0.9770255405548265,1.5962078294172504,0.9748714644047641,1 +1.2548459122000066,1.5344973018844978,-0.09540119743810749,-0.1046553333868987,1.2239469015135578,1 +-3.2090397744472408,0.8441389121874454,2.3538122086767053,3.541566202954287,0.34702734181870465,1 +-3.603602496727503,0.30204738924339203,-0.0585544492972051,0.639791879815238,2.352367745298227,1 +-0.8799527469125623,0.38184389104423133,1.9075472104449394,1.6873971814884494,-1.9013940113781829,0 +1.1536787979977339,0.08946332189273543,0.38311479716547914,0.6813398530821406,-0.43824553187788773,1 +-0.589559297299004,0.3919454845694843,1.4086131057263818,1.3227845261567668,-1.2295252397089191,0 +1.349386722767997,1.2618327501407576,0.9983252674802551,0.2385546136753377,-1.2474886315229026,1 +1.942797695566953,1.9309200576425727,2.7775642977494543,2.7509480561910755,-2.148174680387887,0 +1.002046880004305,-1.892648735870731,0.023012749805542874,0.4113738696354209,-1.8338560526355887,0 +1.7813510200730116,-1.6546684205652724,1.014315067932672,1.3075092018585446,-3.175941418004679,0 +-0.5618537732684308,1.618488805872934,1.012725910739245,0.1447237547702669,-0.38568721169161724,0 +-2.080050480489361,1.5660145076718206,2.8659384999119433,2.3994444192684,-1.7571911585377875,0 +2.071861459031807,1.5819321915413078,0.6842112988791422,0.425860265096731,-0.23197035572082247,0 +0.3782599074721582,-2.078780642357663,-0.7227436643662539,0.9848746921891894,0.5552396910635458,1 +1.6106571673951882,0.8787245690993364,1.1720498431601665,1.88736571994104,-0.3075636336434311,1 +1.7265571284815788,0.6750134277959425,1.71739722520508,1.656506906441229,-2.0735271592389144,1 +-1.8571521720443225,-1.7343727603049683,-0.4627300347016521,1.5857455085718173,1.7071645081701854,1 +-1.077522991542846,-1.2900531295770343,-0.5945495204590028,0.5722603030349935,1.0815956019788695,1 +-2.0928353187159514,0.53968827405994,0.3885701807105375,1.0889614067167963,1.538290681082815,1 +-0.2368554052116022,0.763173959885151,0.2508542640940444,0.44507665567795907,0.75103474546639,1 +-0.17500975830331267,1.1948238951526324,0.6595719123685733,0.9057123596041258,0.7194421170262646,1 +-0.4880058090446492,0.10507398206579455,1.0825129833689229,0.9402872755901576,-1.2137622072325325,0 +-0.5482310609281269,2.5652066694887883,0.43834301824644467,-1.118621033240324,0.5036946977706835,0 +-0.7432873861688771,1.3860043135747833,1.0781373247593005,1.2609407906680994,0.5191844105891736,1 +0.46003939824180007,0.06527853968219599,1.7758095117359025,3.922472647043912,0.09960297978682775,0 +2.0566995922923157,2.8233379447604063,2.5638305003110267,2.12815830733511,-1.480441725656783,0 +-0.3142431574444535,-0.05237636071455276,-0.5807639703942844,-0.9575773951581978,0.34942837549444405,1 +-1.0745076304304773,-1.4429762013880754,1.751184166261124,2.2199155219266613,-2.7151689824538043,0 +0.36485470368561446,-0.39559115176352844,1.26830637169416,2.842861075611598,-0.33874577218613355,1 +-1.2946622420459146,0.9964256947326345,-1.487761315765701,-1.1956975863252444,3.577373337472808,1 +1.0091617893408145,3.367412559926198,2.1766050802419397,1.6388950345670983,-0.22165057689568102,0 +-1.3241562711212087,-0.12930751638603732,2.5346197243227078,3.173811146929783,-2.0866434087025976,0 +-0.8829825865803116,-1.050045421655625,1.9606925364261927,2.5449369288618273,-2.514993952910323,0 +-1.2380174934570043,0.1876570920732401,0.5614162324401284,1.8227400549176533,1.30463796360309,1 +-2.3818980596917485,-0.30886327968594274,1.0235360997939398,2.089295530039154,0.4091333019294171,1 +1.5651112725571985,1.6554161546278303,0.031708271167287305,-0.5208706641715832,0.4870298102594557,1 +0.7269669778919601,-0.6639178795181104,1.3415325792426396,1.5132872963924964,-2.3637160109409985,0 +-1.4025270378771655,2.6653524268381314,1.6014118176556689,1.6487756711326103,1.2325508293181566,1 +1.6015599632242106,0.7549727558261072,-0.7852362540638603,-0.7341058545680181,1.2378402084603441,1 +-1.166657713394523,0.7611010353450893,-0.4986416985261886,-0.3542944875261367,1.9241218237289077,1 +-0.08026108827117384,-1.0551887647072387,2.2203672761710167,2.752306630379624,-3.1651413028501896,0 +-2.096113443515143,0.11368718361993846,2.590565279352208,2.210684498029421,-2.7748948516399574,1 +-0.966071036490391,1.7488895635489248,2.018014337656998,2.256454619537578,-0.13247807411153434,1 +0.12861415343437255,3.3317946794182824,2.0588839677982294,1.840466523779194,0.5338567814090793,1 +-0.5419240082311799,-0.6473695054433745,-0.22635221725592625,0.6051391061767304,0.7258271190494494,1 +-1.5296234487179072,0.969558804188375,2.6470493942589446,2.4718626262828467,-1.950766825813361,0 +-5.043892456948213,-2.74681295732993,-1.5409810893809206,0.7273363894669729,3.324302413409618,1 +-2.433732998487467,3.7642062399118905,1.9361466166729882,-0.8702094791938009,-0.8727418694585812,0 +-0.18860044091502381,2.2716896424978454,-0.03951113263532438,-0.534151969717525,1.8441161592508468,1 +-2.8199504306852257,2.4428426011811064,1.2359908030063336,0.9839361720802553,1.6029105003866508,1 +-0.1615310057905135,-1.4330769848278275,1.1737290771554452,1.8823932255726308,-2.039847744862965,0 +0.13286117321826718,0.07732528396537641,-0.8578806906879226,-1.444775925162166,0.4376029458804199,1 +-1.4466550709733417,-1.8740682926238708,-0.12160616508988303,0.055364844397799096,-1.0494471430472119,0 +2.0797923367508915,2.219685381971847,1.0991206914633669,0.6908645211994979,-0.2687187202798915,0 +-1.6068619653777816,0.8764512389950713,2.3960177761739483,1.6928703374643554,-2.2920675623752054,0 +-2.1594585035736724,-1.6965939809476205,-1.1597192926874123,-0.0753772628669711,1.6385657377017577,1 +0.12077300472544472,-2.2483790492672533,0.8752849092713,1.9238991697722891,-2.2118280265977344,0 +0.34950713997137406,-2.2806240662815633,-0.3839319376785896,1.777298999423122,0.44702007646501807,1 +-0.2654553193635607,-0.46665847180021536,1.1346525343627611,1.3545976890463567,-1.5265641360643252,0 +-1.856611656158378,1.8524465492466156,1.949048149426567,0.932439892243901,-1.0275649872080315,1 +-1.2016280711053537,-0.9902722472058406,-1.0356992565732115,-0.6252810457737985,1.131203827680352,1 +-1.368043876382231,1.1789889846120305,-0.5100459367728748,-0.21309019000612217,2.5926008437243695,1 +0.8942693852357111,2.139071310601736,1.8799228844904594,1.4138011695114332,-0.9721173643339782,0 +-2.4060024939544356,0.9508899388747825,1.9507171769888003,2.7220330455466373,0.219885441850727,1 +-1.1331863863469285,-2.9626780781585627,-0.8011123459938203,1.9250101376335575,1.3938449167471199,1 +-2.647914324599812,-1.3610202151019353,-1.3348239104925301,-0.4235076276249292,2.162772596237362,1 +-0.055140663699368764,1.2881848437048036,1.5290617851992776,1.5559137520965856,-0.5338645841188343,0 +-1.6546110439720927,-0.025686262282198058,-0.017499025571771964,0.6497472140291349,1.2855194211162537,1 +2.975574761230523,2.419820582192545,0.6785516902215438,0.03850874460031484,-0.1086097527953227,0 +1.0763513351130107,2.1294094611406873,1.2079760885374984,0.8249951886323986,-0.12813505345106035,0 +-1.591954595850689,-0.31830491433761376,-0.20804561224971763,0.47376610897359195,1.2206523729895955,1 +-0.5663335747054519,-0.7901233485646082,0.19897067861984086,1.636826317019823,0.7376159657800063,1 +-0.6472819629840465,-1.3938145754481424,-1.078652842488317,-0.8949624948329402,0.3436757090900173,1 +2.3964289711656575,1.3840583004891485,0.05674820503788647,-0.36640690353630334,0.04800716939780858,0 +-1.5879749726823809,1.4097794209321337,1.9999909308327104,1.571841285517877,-0.97578654146721,0 +-2.0608784170949264,-1.9292752321736781,-2.600790564303219,-1.276013658868342,3.4024582518300086,1 +0.6360578412575373,2.651867689418183,-0.15325948626672936,-0.7106250954850648,2.01885855535229,1 +1.1063828414340589,0.8691778993427344,0.6049688452042938,0.2859114573080176,-0.5923514635433662,0 +-1.0584031106409864,1.892100113913194,1.9215988006883913,1.3792377301184393,-0.6996833151518032,0 +-3.322033044786915,-1.4002695991049532,-1.6410013316373464,-1.2604556973060097,2.140143987446071,1 +-1.8763315768478992,0.2764298476750694,0.948597969426878,2.1015798196408677,1.0141053190223035,1 +-1.0291014407041037,-0.02065552711684937,3.8827450770240564,4.385191955998962,-3.8788530177379528,0 +0.6875135596368191,-2.5821662620414925,-1.3972523899406486,0.08643189645327709,0.5266243319040311,1 +0.7128506438784368,5.737793131621844,2.270182182523157,0.8842600520778051,1.205114885288078,1 +-2.552191918547829,0.9978962939519059,1.0137223518719514,1.5274720692791357,1.180294132547354,1 +-0.25950429659657837,2.606343069201997,0.4811422093989864,-0.7456597418429676,0.7592336830283972,0 +0.6119053248109316,1.032538881515911,0.22549898505907245,-0.03447961375704134,0.26721393410869365,0 +-1.3343520113214837,0.6556699815298489,1.7419807984145028,1.3514252678511829,-1.4593073086303021,0 +-1.8332962405846545,1.2603644391209619,2.29984563806238,1.0962541438293516,-2.264563429080373,0 +-0.04551418261958373,1.1359760557676202,1.6828664813100631,1.4547604010169624,-1.1588624216666135,0 +-1.5141736970725765,-1.487787355228758,-0.9555348395511414,0.5311916604468465,1.8243978856794336,1 +2.7974314238344116,0.9879068057949466,-0.575225216168288,-1.0997597619749349,0.17896071281217574,0 +0.5607124807115611,-0.4499914267628853,0.45555425739311306,1.8494412145170505,0.33775024286845745,1 +-0.8227790111401876,-0.1419745159858632,1.797990801211173,2.062431188491476,-1.7777878659828437,0 +-1.3021306380180802,2.7445906607932438,1.7914988948026433,1.8207365854764395,1.0254696023941723,1 +-1.6751532434457437,-1.0594884596578384,0.2019177616490525,1.9402622556553806,1.1668499188225656,1 +0.5090246062619943,1.2462977627755303,1.888570960302307,1.6835622311737195,-1.4608607071889814,0 +-2.9213691325281457,-1.2738835801340296,-0.6406463428319629,0.528208217266391,1.7750375295167942,1 +-0.8882140219876686,0.8607145646432016,1.6762960658564352,1.311419279698875,-1.2946758951498674,0 +-0.9483225547090204,-0.5662366331511879,1.6018056835444354,1.9950467368749156,-1.7787768515802276,0 +-0.7318007391528466,2.0407903746494496,1.1044514574069897,0.4893977241783146,0.2608261345698315,0 diff --git a/readme.md b/readme.md index 13f8500..3a84a6b 100644 --- a/readme.md +++ b/readme.md @@ -12,7 +12,7 @@ A decision-making framework for the cost-efficient design of experiments, balanc code ### Static experimental designs -Here we assume that the same experimental design will be used for a population of examined entities, hence the word 'static'. +Here we assume that the same experimental design will be used for a population of examined entities, hence the word "static". For each subset of experiments, we consider an estimate of the value of acquired information. To give an example, if a set of experiments is used to predict the value of a specific target variable, our framework can leverage a built-in integration with [MLJ.jl](https://github.com/alan-turing-institute/MLJ.jl) to estimate predictive accuracies of machine learning models fitted over subset of experimental features. @@ -26,7 +26,7 @@ Assuming the information values and optimized experimental costs for each subset We consider 'personalized' experimental designs that dynamically adjust based on the evidence gathered from the experiments. This approach is motivated by the fact that the value of information collected from an experiment generally differs across subpopulations of the entities involved in the triage process. -At the beginning of the triage process, an entity's prior data is used to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model likely experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts (sampled from a generative model, hence the word 'generative'), we adjust the continuation based on this evidence. +At the beginning of the triage process, an entity's prior data is used to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model likely experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts (sampled from a generative model, hence the word "generative"), we adjust the continuation based on this evidence. code diff --git a/src/GenerativeDesigns/EfficientValueMDP.jl b/src/GenerativeDesigns/EfficientValueMDP.jl index c6df177..18adfc5 100644 --- a/src/GenerativeDesigns/EfficientValueMDP.jl +++ b/src/GenerativeDesigns/EfficientValueMDP.jl @@ -139,7 +139,7 @@ Internally, an instance of the `EfficientValueMDP` structure is created and a su (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); value = (evidence, costs) -> (1 - uncertainty(evidence) + 0.005 * sum(costs)); diff --git a/src/GenerativeDesigns/GenerativeDesigns.jl b/src/GenerativeDesigns/GenerativeDesigns.jl index 9c3b1c1..037714b 100644 --- a/src/GenerativeDesigns/GenerativeDesigns.jl +++ b/src/GenerativeDesigns/GenerativeDesigns.jl @@ -14,7 +14,7 @@ using MCTS using ..CEEDesigns: front export UncertaintyReductionMDP, DistanceBased -export QuadraticDistance, DiscreteDistance, MahalanobisDistance +export QuadraticDistance, DiscreteDistance, SquaredMahalanobisDistance export Exponential export Variance, Entropy export Evidence, State diff --git a/src/GenerativeDesigns/UncertaintyReductionMDP.jl b/src/GenerativeDesigns/UncertaintyReductionMDP.jl index ce37d8d..b57127b 100644 --- a/src/GenerativeDesigns/UncertaintyReductionMDP.jl +++ b/src/GenerativeDesigns/UncertaintyReductionMDP.jl @@ -184,7 +184,7 @@ In the uncertainty reduction setup, minimize the expected experimental cost whil (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); # initialize evidence @@ -315,7 +315,7 @@ Internally, an instance of the `UncertaintyReductionMDP` structure is created fo (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); # initialize evidence diff --git a/src/GenerativeDesigns/distancebased.jl b/src/GenerativeDesigns/distancebased.jl index fa10cb9..69c5c81 100644 --- a/src/GenerativeDesigns/distancebased.jl +++ b/src/GenerativeDesigns/distancebased.jl @@ -5,12 +5,17 @@ This returns an anonymous function `(x, col; prior) -> λ * (x .- col).^2 / σ`. If `standardize` is set to `true`, `σ` represents `col`'s variance calculated in relation to `prior`, otherwise `σ` equals one. """ -QuadraticDistance(; λ = 1, standardize = true) = +function QuadraticDistance(; λ = 1, standardize = true) + σ = nothing + function (x, col; prior = ones(length(col))) - σ = standardize ? var(col, Weights(prior); corrected = false) : 1 + if isnothing(σ) + σ = standardize ? var(col, Weights(prior); corrected = false) : 1 + end return λ * (x .- col) .^ 2 / σ end +end """ DiscreteDistance(; λ=1) @@ -30,16 +35,21 @@ Return an anonymous function `x -> exp(-λ * sum(x; init=0))`. Exponential(; λ = 1 / 2) = x -> exp(-λ * x) # default uncertainty functionals -compute_variance(data::AbstractVector; weights) = var(data, Weights(weights)) +function compute_variance(data::AbstractVector; weights) + var(data, Weights(weights); corrected = false) +end -compute_variance(data; weights) = sum(var(Matrix(data), Weights(weights), 1)) +function compute_variance(data; weights) + sum(var(Matrix(data), Weights(weights), 1; corrected = false)) +end """ - Variance(data; prior) + Variance() -Return a function of `weights` that computes the fraction of variance in the data, relative to the variance calculated with respect to a specified `prior`. +Return a function of `(data; prior)`. When this function is called as part of an instantiation procedure in [`DistanceBased`](@ref), +it returns an internal function of `weights` that computes the fraction of variance in the data, relative to the variance calculated with respect to a specified `prior`. """ -function Variance(data; prior) +Variance() = function (data; prior) initial = compute_variance(data; weights = prior) return weights -> (compute_variance(data; weights) / initial) end @@ -50,14 +60,17 @@ function compute_entropy(labels; weights) end """ - Entropy(labels; prior) + Entropy() -Return a function of `weights` that computes the fraction of information entropy, relative to the entropy calculated with respect to a specified `prior`. +Return a function of `(labels; prior)`. When this function is called as part of an instantiation procedure in [`DistanceBased`](@ref), +it returns an internal function of `weights` that computes the fraction of information entropy, relative to the entropy calculated with respect to a specified `prior`. """ -function Entropy(labels; prior) - @assert elscitype(labels) <: Multiclass "labels must be of `Multiclass` scitype, but `elscitype(labels)=$(elscitype(labels))`" - initial = compute_entropy(labels; weights = prior) - return (weights -> compute_entropy(labels; weights) / initial) +function Entropy() + function (labels; prior) + @assert elscitype(labels) <: Multiclass "labels must be of `Multiclass` scitype, but `elscitype(labels)=$(elscitype(labels))`" + initial = compute_entropy(labels; weights = prior) + return weights -> (compute_entropy(labels; weights) / initial) + end end # Return a function that calculates the sum of distances in each row, column-wise, and applies weights based on the prior. @@ -84,51 +97,61 @@ function sum_of_distances(data::DataFrame, targets::Vector, distances; prior::We end """ - MahalanobisDistance(; diagonal=0) + SquaredMahalanobisDistance(; diagonal=0) -Returns a function that computes [Mahalanobis distance](https://en.wikipedia.org/wiki/Mahalanobis_distance) between each row of `data` and the evidence. +Returns a function that computes [squared Mahalanobis distance](https://en.wikipedia.org/wiki/Mahalanobis_distance) between each row of `data` and the evidence. For a singular covariance matrix, consider adding entries to the matrix's diagonal via the `diagonal` keyword. +To accommodate missing values, we have implemented an approach described in https://www.jstor.org/stable/3559861, on page 285. + # Arguments - - `diagonal`: A scalar or vector to be added to the diagonal entries of the covariance matrix. + - `diagonal`: A scalar to be added to the diagonal entries of the covariance matrix. # Returns It returns a high-level function of `(data, targets, prior)`. -When called, that function will return an internal function `compute_distances` that takes an `Evidence` and computes the Mahalanobis distance based on the input data and the evidence. +When called, that function will return an internal function `compute_distances` that takes an `Evidence` and computes the squared Mahalanobis distance based on the input data and the evidence. """ -function MahalanobisDistance(; diagonal = 0) +function SquaredMahalanobisDistance(; diagonal = 0) function (data, targets, prior) non_targets = setdiff(names(data), targets) if !all(t -> t <: Real, eltype.(eachcol(data[!, non_targets]))) @warn "Not all column types in the predictor matrix are numeric ($(eltype.(eachcol(data)))). This may cause errors." end - Σ = cov(Matrix(data[!, non_targets]), Weights(prior)) - # add diagonal entries - diagonal = diagonal isa Number ? fill(diagonal, size(Σ, 1)) : diagonal - foreach(i -> Σ[i, i] += diagonal[i], axes(Σ, 1)) - # get the inverse of Σ - Λ = inv(Σ) + Λ = Dict( + Set(features) => begin + Σ = cov(Matrix(data[!, features]), Weights(prior); corrected = false) + # Add diagonal entries. + foreach(i -> Σ[i, i] += diagonal, axes(Σ, 1)) + + inv(Σ) + end for features in powerset(non_targets, 1, length(non_targets)) + ) compute_distances = function (evidence::Evidence) - if isempty(evidence) + evidence_keys = collect(keys(evidence) ∩ non_targets) + + if length(evidence_keys) == 0 return zeros(nrow(data)) - else - vec_evidence = map(colname -> get(evidence, colname, 0), non_targets) - distances = map(eachrow(data)) do row - vec_row = map( - colname -> haskey(evidence, colname) ? row[colname] : 0, - non_targets, - ) - - z = vec_evidence - vec_row - dot(z, Λ * z) - end + end + + vec_evidence = map(k -> evidence[k], evidence_keys) - return distances + factor_p_q = length(non_targets) / length(evidence_keys) + distances = map(eachrow(data)) do row + # We retrieve the precomputed inverse of the covariance matrix coresponding to the "observed part" + # See #12 and, in particular, https://www.jstor.org/stable/3559861 + Λ_marginal = Λ[Set(evidence_keys)] + + factor_p_q * dot( + (vec_evidence - Vector(row[evidence_keys])), + Λ_marginal * (vec_evidence - Vector(row[evidence_keys])), + ) end + + return distances end return compute_distances @@ -136,11 +159,11 @@ function MahalanobisDistance(; diagonal = 0) end """ - DistanceBased(data; target, uncertainty=Entropy, similarity=Exponential(), distance=Dict(); prior=ones(nrow(data))) + DistanceBased(data; target, uncertainty=Entropy(), similarity=Exponential(), distance=Dict(); prior=ones(nrow(data))) Compute distances between experimental evidence and historical readouts, and apply a 'similarity' functional to obtain probability mass for each row. -Consider using [`QuadraticDistance`](@ref), [`DiscreteDistance`](@ref), and [`MahalanobisDistance`](@ref). +Consider using [`QuadraticDistance`](@ref), [`DiscreteDistance`](@ref), and [`SquaredMahalanobisDistance`](@ref). # Return value @@ -168,7 +191,7 @@ A named tuple with the following fields: (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); ``` @@ -176,7 +199,7 @@ A named tuple with the following fields: function DistanceBased( data::DataFrame; target, - uncertainty = Variance, + uncertainty = Variance(), similarity = Exponential(), distance = Dict(), prior = ones(nrow(data)), @@ -215,11 +238,19 @@ function DistanceBased( compute_weights = function (evidence::Evidence) similarities = prior .* map(x -> similarity(x), compute_distances(evidence)) - # hard match on target columns + # Perform hard match on target columns. for colname in collect(keys(evidence)) ∩ targets similarities .*= data[!, colname] .== evidence[colname] end + # If all similarities were zero, the `Weights` constructor would error. + if sum(similarities) ≈ 0 + similarities .= 1 + for colname in collect(keys(evidence)) ∩ targets + similarities .*= data[!, colname] .== evidence[colname] + end + end + return Weights(similarities ./ sum(similarities)) end diff --git a/src/StaticDesigns/scripts/demo_literate.jl b/src/StaticDesigns/scripts/demo_literate.jl index 817f12d..3c48370 100644 --- a/src/StaticDesigns/scripts/demo_literate.jl +++ b/src/StaticDesigns/scripts/demo_literate.jl @@ -21,7 +21,7 @@ data = CSV.File("data/heart_disease.csv") |> DataFrame # ## Predictive Accuracy -# The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +# The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. # We specify the experiments along with the associated features: diff --git a/src/fronts.jl b/src/fronts.jl index 4153b47..f33e742 100644 --- a/src/fronts.jl +++ b/src/fronts.jl @@ -105,9 +105,7 @@ function plot_front( fontsize = 16, ) for i = 2:length(designs) - if xs[i] < 10_000 - scatter!(p, [xs[i]], [ys[i]]; label = labels[i], c = grad[i], mscolor = nothing) - end + scatter!(p, [xs[i]], [ys[i]]; label = labels[i], c = grad[i], mscolor = nothing) end return p @@ -132,12 +130,20 @@ function make_labels(designs) end """ - plot_evals(eval; ylabel="information measure") + plot_evals(evals; f, ylabel="information measure") Create a stick plot that visualizes the performance measures evaluated for subsets of experiments. + +Argument `evals` should be the output of [`evaluate_experiments`](@ref CEEDesigns.StaticDesigns.evaluate_experiments) and the kwarg `f` (if provided) is a function that +should take as input `evals` and return a list of its keys in the order to be plotted on the x-axis. +By default they are sorted by length. """ -function plot_evals(evals; ylabel = "information measure", kwargs...) - xs = sort!(collect(keys(evals)); by = x -> length(x)) +function plot_evals( + evals; + ylabel = "information measure", + f = x -> sort(collect(keys(x)); by = length), +) + xs = f(evals) ys = map(xs) do x return evals[x] isa Number ? evals[x] : evals[x].loss end diff --git a/test/GenerativeDesigns/test.jl b/test/GenerativeDesigns/test.jl index 86f0825..73d21e1 100644 --- a/test/GenerativeDesigns/test.jl +++ b/test/GenerativeDesigns/test.jl @@ -1,5 +1,5 @@ # test with the sum distances include("test_distances_sum.jl") -# test with the Mahalanobis distance +# test with the squared Mahalanobis distance include("test_mahalanobis.jl") diff --git a/test/GenerativeDesigns/test_distances_sum.jl b/test/GenerativeDesigns/test_distances_sum.jl index fc0b1ac..013a304 100644 --- a/test/GenerativeDesigns/test_distances_sum.jl +++ b/test/GenerativeDesigns/test_distances_sum.jl @@ -27,7 +27,7 @@ evidence = Evidence("Age" => 35, "Sex" => "M") r = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); @test all(x -> hasproperty(r, x), [:sampler, :uncertainty, :weights]) diff --git a/test/GenerativeDesigns/test_mahalanobis.jl b/test/GenerativeDesigns/test_mahalanobis.jl index 6222834..f7ea57c 100644 --- a/test/GenerativeDesigns/test_mahalanobis.jl +++ b/test/GenerativeDesigns/test_mahalanobis.jl @@ -25,9 +25,9 @@ evidence = Evidence() r = DistanceBased( data; target = "HeartDisease", - uncertainty = Variance, + uncertainty = Variance(), similarity = Exponential(; λ = 5), - distance = MahalanobisDistance(; diagonal = 1), + distance = SquaredMahalanobisDistance(; diagonal = 1), ); @test all(x -> hasproperty(r, x), [:sampler, :uncertainty, :weights]) (; sampler, uncertainty, weights) = r diff --git a/test/runtests.jl b/test/runtests.jl index 808e730..dabbc62 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,10 +2,10 @@ using SafeTestsets, BenchmarkTools @time begin @time @safetestset "fronts tests" begin - #include("fronts.jl") + include("fronts.jl") end @time @safetestset "`StaticDesigns` tests" begin - #include("StaticDesigns/test.jl") + include("StaticDesigns/test.jl") end @time @safetestset "`GenerativeDesigns` tests" begin include("GenerativeDesigns/test.jl") diff --git a/tutorials/GenerativeDesigns.jl b/tutorials/GenerativeDesigns.jl index d74f31f..2d1f4f1 100644 --- a/tutorials/GenerativeDesigns.jl +++ b/tutorials/GenerativeDesigns.jl @@ -1,90 +1,11 @@ # # Heart Disease Triage Meets Generative Modeling # Consider again a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer; instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. +# For details on the theoretical background and notation, please see our tutorial on [generative experimental designs](SimpleGenerative.md). This tutorial +# is a concrete application of the tools described in that document. # Importantly, we aim to design personalized adaptive policies for each patient. At the beginning of the triage process, we use a patient's prior data, such as sex, age, or type of chest pain, to project a range of cost-efficient experimental designs. Internally, while constructing these designs, we incorporate multiple-step-ahead lookups to model probable experimental outcomes and consider the subsequent decisions for each outcome. Then after choosing a specific decision policy from this set and acquiring additional experimental readouts, we adjust the continuation based on this evidence. -# The personalized experimental designs are motivated by the fact that the value of information collected from an experiment often differs across subsets of the entities involved in the triage process. - -# ![information value matrix](assets/information_value_matrix.png) - -# In the context of static designs, where each individual from the 'Population' undergoes the same triage process, 'Experiment C' would contribute the maximum information value. On the other hand, if we have the capability to adapt the triage process specifically for 'Population 1' and 'Population 2', we would choose 'Experiment 1' and 'Experiment 2' respectively, thereby improving the value of information acquired. - -# ## Theoretical Framework - -# Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$, and let $y$ denote the target variable that we want to predict. - -# We conceptualize the triage as a Markov decision process, in which we iteratively choose to conduct a subset of experiments $S \subseteq E$ and then, based on the experimental evidence, update our belief about the distribution of outcomes for the experiments that have not yet been conducted. - -# Within the framework, -# - _state_ is modeled as the set of experiments conducted so far along with the acquired experimental evidence and accumulated costs; -# - _actions_ are subsets of experiments that have not yet been conducted; the size of these subsets is restricted by the maximum number of parallel experiments. - -# Importantly, the outcome of a set $S$ of experiments is modeled as a random variable $e_S$, conditioned on the current state, i.e., combined evidence. This means that if in a given state outcomes from experiments in $S \subseteq E$ are available, the outcome of experiments in $S' \subseteq E \setminus S$ is drawn from a posterior $r \sim q(e_{S'} | e_S)$. - -# We do not claim to know the 'best' way to define the posterior $q$. Instead, our approach is generic and allows us to consider any generative function that takes the current state and the set of experiments to be conducted, and returns a sample drawn from the implicit, theoretical posterior of the selected experiments. - -# The information value associated with the state, derived from experimental evidence, can be modeled through any statistical or information-theoretic measure such as the variance or uncertainty associated with the target variable posterior $q(y|e_S)$. - -# For example, consider [EDDI: Efficient Dynamic Discovery of High-Value Information with Partial VAE](https://arxiv.org/pdf/1809.11142.pdf), where the authors use a VAE approach to model the posterior distribution and utilize Kullback-Leibler divergence to model the information gain. - -# #### Distance-Based Sampling from Historical Data - -# We implemented an approach that iteratively distributes a belief over historical entities likely to be similar to the entity currently being tested. As a result, we sample from the outputs of these historical entities, weighted by the probabilistic weight assigned by our system of belief. In doing so, we produce a specific model for $q(e_{S^{'}} | e_S)$ and $q(y | e_S)$. - -# More specifically, consider a dataset of historical outputs across $m$ features $X = \{x_1, \ldots, x_m\}$ for $l$ entities (with entities and features representing rows and columns, respectively). Let's denote the target variable we want to predict as $y$. - -# We assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. - -# In what follows, we discuss the assignment of probabilistic weights $w_j$ to each entity (or row in the dataset). - -# In the initial state, when there is no experimental evidence gathered yet, we assign the weights according to a predetermined prior. - -# If this is not the case, we have to compute the weights using particular distance and similarity functionals. - -# For each feature $x\in X$, we consider a function $\rho_x$, which measures the distance between two outputs. By default, we consider: -# - Rescaled Kronecker delta (i.e., $\rho(x, y)=0$ only when $x=y$, and $\rho(x, y)= \lambda$ under any other circumstances, with $\lambda > 0$) for discrete features (i.e., features whose types are modeled as `MultiClass` type in [ScientificTypes.jl](https://github.com/JuliaAI/ScientificTypes.jl)); -# - Rescaled squared distance $\rho(x, y) = λ \frac{(x - y)^2}{2\sigma_2}$, where $\sigma_2$ is the variance of the feature column, estimated with respect to the prior for continuous features. - -# Therefore, given an experimental state with readouts over the feature set $F \subseteq X$, we can calculate the total distance from the entity recorded in the $j$-th row as $d_j = \sum_{x\in F} \rho_x (\hat x, x_j)$, where $\hat x$ and $x_j$ denote the readout for feature $x$ for the entity being tested and the entity recorded in $j$-th column, respectively. - -# Alternatively, we could use the [Mahalanobis distance](https://en.wikipedia.org/wiki/Mahalanobis_distance#Definition). - -# Next, we convert distances $d_j$ into probabilistic weights $w_j$. By default, we use a rescaled exponential function, i.e., we put $w_j = \exp(-\lambda d_j)$ for some $\lambda>0$. Notably, $\lambda$'s value determines how belief is distributed across the historical entities. Larger values of $\lambda$ concentrate the belief tightly around the 'closest' historical entities, while smaller values distribute more belief to more distant entities. - -# Importantly, the proper choice of the distance functionals and the 'similarity' functional discussed above is a question of hyper-optimization. - -# Assuming the weights $w_j$ have been assigned, we can sample an index $\hat j \in \{ 1, \ldots, l\}$ according to these weights. To draw a sample $\hat x$ from feature $x\in X$, we let $\hat x$ be equal to the value in the column associated with feature $x$ for the $\hat j$-th row (entity). We can also take a sample from the target variable in the same way. - -# ### Objective Sense -# The reward and stopping condition of the triage process can be interpreted in various ways. -# - The triage may continue until the uncertainty about the posterior distribution of the target variable falls below a certain level. Our aim is to minimize the anticipated combined monetary cost and execution time of the triage (considered as a 'negative' reward). If all experiments are conducted without reaching below the required uncertainty level, or if the maximum number of experiments is exceeded, we penalize this scenario with a 'minus infinite' reward. -# - We may aim to minimize the expected uncertainty while being constrained by the costs of the experiment. -# - Alternatively, we could maximize the value of experimental evidence, adjusted for the incurred experimental costs. - -# ### Policy Search -# Standard MDP algorithms can be used to solve this problem (offline learning) or construct the policy (online learning) for the sequential decision-making. - -# Our MDP's state space is finite-dimensional but generally continuous due to the allowance of continuous features, which complicates the problem and few algorithms specialize in this area. - -# We used the Double Progressive Widening Algorithm for this task as detailed in [A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation](https://arxiv.org/abs/1405.5498). - -# In a nutshell, the Double Progressive Widening (DPW) algorithm is designed for online learning in complex environments, particularly those known as Continuous Finite-dimensional Markov Decision Processes where the state space is continuous. The key idea behind DPW is to progressively expand the search tree during the Monte Carlo Tree Search (MCTS) process. The algorithm does so by dynamically and selectively adding states and actions to the tree based on defined heuristics. - -# In the context of online learning, this algorithm addresses the complexity and challenges of real-time decision-making in domains with a large or infinite number of potential actions. As information is gathered in actual runtime, the algorithm explores and exploits this information to make optimal or near-optimal decisions. In other words, DPW permits the learning process to adapt on-the-fly as more data is made available, making it an effective tool for the dynamic and uncertain nature of online environments. - -# In particular, at the core of the Double Progressive Widening (DPW) algorithm are several key components, including expansion, search, and rollout. - -# The search component is where the algorithm sifts through the search tree to determine the most promising actions or states to explore next. By using exploration-exploitation strategies, it can effectively balance its efforts between investigating previously successful actions and venturing into unexplored territories. - -# The expansion phase is where the algorithm grows the search tree by adding new nodes, representing new state-action pairs, to the tree. This is done based on a predefined rule that dictates when and how much the tree should be expanded. This allows the algorithm to methodically discover and consider new potential actions without becoming overwhelmed with choices. - -# Lastly, the rollout stage, also known as a simulation stage, is where the algorithm plays out a series of actions to the end of a game or scenario using a specific policy (like a random policy). The results of these rollouts are then used to update the estimates of the values of state-action pairs, increasing the accuracy of future decisions. - -# ![One iteration of the MCTS algorithm, taken from https://ieeexplore.ieee.org/document/6145622](assets/mcts.png) - -# In the above figure, nodes represent states of the decision process, while edges correspond to actions connecting these states. - # ## Heart Disease Dataset # In this tutorial, we consider a dataset that includes 11 clinical features along with a binary variable indicating the presence of heart disease in patients. The dataset can be found at [Kaggle: Heart Failure Prediction](https://www.kaggle.com/datasets/fedesoriano/heart-failure-prediction). It utilizes heart disease datasets from the [UCI Machine Learning Repository](https://archive.ics.uci.edu/dataset/45/heart+disease). @@ -120,7 +41,7 @@ using CEEDesigns, CEEDesigns.GenerativeDesigns # As previously discussed, we provide a dataset of historical records, the target variable, along with an information-theoretic measure to quantify the uncertainty about the target variable. # In what follows, we obtain three functions: -# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator, # - `uncertainty`: this is a function of `evidence`, # - `weights`: this represents a function of `evidence` that distributes probabilistic weights across the rows in the dataset. @@ -131,7 +52,7 @@ using CEEDesigns, CEEDesigns.GenerativeDesigns (; sampler, uncertainty, weights) = DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), ); @@ -145,7 +66,7 @@ categorical_feats = setdiff(names(data), numeric_feats) DistanceBased( data; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), distance = merge( Dict(c => DiscreteDistance() for c in categorical_feats), @@ -153,14 +74,14 @@ DistanceBased( ), ); -# You can also use the Mahalanobis distance (`MahalanobisDistance(; diagonal)`). For example, we could write: +# You can also use the squared Mahalanobis distance (`SquaredMahalanobisDistance(; diagonal)`). As the squared Mahalanobis distance only works with numeric features, we have to select a few, along with the target variable. For example, we could write: DistanceBased( - data[!, ["RestingBP", "MaxHR", "Cholesterol", "FastingBS", "HeartDisease"]]; # the Mahalanobis distance only works with numeric features, so we selected a few, along with the target variable + data[!, ["RestingBP", "MaxHR", "Cholesterol", "FastingBS", "HeartDisease"]]; target = "HeartDisease", - uncertainty = Entropy, + uncertainty = Entropy(), similarity = Exponential(; λ = 5), - distance = MahalanobisDistance(; diagonal = 1), + distance = SquaredMahalanobisDistance(; diagonal = 1), ); # The package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. @@ -268,7 +189,7 @@ experiments = Dict( "HeartDisease" => (100.0, 100.0), ) -# We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. +# We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time. In our case, we aim to minimize the execution time. ## minimize time, two concurrent experiments at maximum seed!(1) @@ -297,7 +218,7 @@ plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") # In this experimental setup, we aim to maximize the value of experimental evidence, adjusted for the incurred experimental costs. -# For this purpose, we need to specify a function that quantifies the 'value' of decision-process making state, modeled as a tuple of experimental evidence and costs. +# For this purpose, we need to specify a function that quantifies the "value" of decision-process making state, modeled as a tuple of experimental evidence and costs. value = function (evidence, (monetary_cost, execution_time)) return (1 - uncertainty(evidence)) - (0.005 * sum(monetary_cost)) diff --git a/tutorials/Project.toml b/tutorials/Project.toml index d4a0fc7..616611c 100644 --- a/tutorials/Project.toml +++ b/tutorials/Project.toml @@ -2,6 +2,8 @@ BetaML = "024491cd-cc6b-443e-8034-08ea7eb7db2b" CEEDesigns = "e939450b-799e-4198-a5f5-3f2f7fb1c671" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" +Copulas = "ae264745-0b69-425e-9d9d-cf662c5eec93" D3Trees = "e3df1716-f71e-5df9-9e2d-98e193103c45" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DisplayAs = "0b91fe84-8a4c-11e9-3e1d-67c38462b6d6" @@ -14,7 +16,6 @@ Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" MCTS = "e12ccd36-dcad-5f33-8774-9175229e7b33" MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" MLJModels = "d491faf4-2d78-11e9-2867-c94bc002c0b7" -POMDPSimulators = "e0d0a172-29c6-5d4e-96d0-f262df5d01fd" POMDPTools = "7588e00f-9cae-40de-98dc-e0c70c48cdd7" POMDPs = "a93abf59-7444-517b-a68a-c42f96afdd7d" PlotlyBase = "a03496cd-edff-5a9b-9e67-9cda94a718b5" @@ -24,3 +25,4 @@ Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781" PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd" diff --git a/tutorials/SimpleGenerative.jl b/tutorials/SimpleGenerative.jl new file mode 100644 index 0000000..12fc1d4 --- /dev/null +++ b/tutorials/SimpleGenerative.jl @@ -0,0 +1,515 @@ +# # Generative Experimental Designs + +# This document describes the theoretical background behind tools in CEEDesigns.jl for generative experimental designs +# and demonstrates uses on synthetic data examples. + +# ## Setting + +# Generative experimental designs differ from static designs in that the experimental design is specific for (or "personalized") to an incoming entity. +# Personalized cost-efficient experimental designs for the new entity are made based on existing information (if any) about the new entity, +# and from posterior distributions of unobserved features of that entity conditional on its observed features and historical data. +# The entities may be patients, molecular compounds, or any other objects where one has a store of historical data +# and seeks to find efficient experimental designs to learn more about a new arrival. +# +# The personalized experimental designs are motivated by the fact that the value of information collected from an experiment +# often differs across subsets of the population of entities. + +# ![information value matrix](assets/information_value_matrix.png) + +# In the context of static designs, we do not aspire to capture variation in information gain across different entities. Instead, we assume all entities come from a "Population" with a uniform information gain, in which case "Experiment C" would provide the maximum information value. +# On the other hand, if we have the ability to discern if the entity belong to subpopulations "Population 1" or "Population 2," then we can tailor our +# design to suggest either "Experiment A" or "Experiment B." Clearly, in the limit of a maximally heterogenous population, each +# entity has its own "row." Our tools are able to handle the entire spectrum of such scenarios though distance based similarity, +# described below. + +# ## Theoretical Framework + +# ### Historical Data + +# Like static designs, we consider a set $E$ of $n$ experiments, each with an associated tuple $(m_{e},t_{e})$ of monetary and +# time costs (for more details on experiments and arrangements, see the tutorial on [static experimental designs](SimpleStatic.md)). +# +# Additionally, consider a historical dataset giving measurements on $m$ features $X = \{x_1, \ldots, x_m\}$ for $l$ entities +# (with entities and features representing rows and columns, respectively). Each experiment $e$ may yield measurements on some +# subset of features $(X_{e}\subseteq X)$. +# +# Furthermore there is an additional column $y$ which is a target variable we want to predict (CEEDesigns.jl may allow $y$ +# to be a vector, but we assume it is scalar here for ease of presentation). +# +# Letting $m=3$, then the historical dataset may be visualized as the following table: + +# ![historical data](assets/generative_historical.png) + +# ### New Entity + +# When a new entity arrives, it will have an unknown outcome $y$ and unknown values of some or all features $x_{i} \in X$. +# We call the _state_ of the new entity the set of experiments conducted upon it so far (if any), along with the +# measurements on any features produced by those experiments (if any), called _evidence_. +# +# Consider the case where there are $n=3$ experiments, each of which yields a measurement on a single feature. Then +# the state of a new entity arriving for which $e_{1}$ has already been performed will look like: + +# ![state](assets/generative_state.png) + +# Letting $S$ denote the set of experiments which have been performed so far, then $S^{\prime}=E\S$ are unperformed experiments. +# Then, _actions_ one can perform to learn more about the new entity are subsets of $S^{\prime}$. The size of subsets +# is limited by the maximum number of parallel experiments. +# +# In our running example, if the maximum number of parallel experiments is at least 2, then the available actions are: + +# ![actions](assets/generative_actions.png) + +# ### Posterior Distributions + +# Let $e_{S}$ be a random variable denoting the outcome of some experiments $S$ on the new entity. Then, there exists a +# posterior distribution $q(e_{S^{\prime}}|e_{S})$ over outcomes of unperformed experiments $S^{\prime}$, conditioned on the current +# state. +# +# Furthermore, there also exists a posterior distribution over the unobserved target variable $q(y|e_{S})$. The information +# value of the current state, derived from experimental evidence, can be defined as any statistical or information-theoretic +# measure applied to $q(y|e_{S})$. This can include variance or entropy (among others). The information value +# of the set of experiments performed in $S$ is analogous to $v_{S}$ defined in [static experimental designs](SimpleStatic.md). + +# ### Similarity Weighting + +# There may be many ways to define these posterior distributions, but our approach uses distance-based similarity +# scores to construct weights $w_{j}$ over historical entities which are similar to the new entity. These weights can be used to +# weight the values of $y$ or features associated with $e_{S^{\prime}}$ to construct approximations of $q(y|e_{S})$ and +# $q(e_{S^{\prime}}|e_{S})$. +# +# If there is no evidence associated with a new entity, we assign $w_{j}$ according to some prior distribution (uniform by default). +# Otherwise we use some particular distance and similarity metrics. +# +# For each feature $x\in X$, we consider a function $\rho_x$, which measures the distance between two outputs. Please be aware that the "distances" discussed here do not conform to the mathematical definition of "metrics", even though they are functions derived from underlying metrics (i.e., a square of a metric). This is justified when considering how these "distances" are subsequently converted into probabilistic weights. +# +# By default, we consider the following distances: +# - Rescaled Kronecker delta (i.e., $\rho(x, y)=0$ only when $x=y$, and $\rho(x, y)= \lambda$ under any other circumstances, with $\lambda > 0$) for discrete features (i.e., features whose types are modeled as `MultiClass` type in [ScientificTypes.jl](https://github.com/JuliaAI/ScientificTypes.jl)); +# - Rescaled ("standardized", by default) squared distance $\rho(x, y) = λ \frac{(x - y)^2}{\sigma^2}$, where $\sigma^2$ is the variance of the feature column, estimated with respect to the prior for continuous features. +# - Squared Mahalanobis distance $\rho(x,y) = (x-y)^{⊤}\Sigma^{-1}(x-y)$, where $\Sigma$ is the empirical covariance matrix of the historical data. +# +# For distance metrics assuming independence of features (Kronecker delta and squared distance), given the new entity's experimental state with readouts over the feature set $F = \bigcup X_{e}$, where $e \in S$, we can calculate +# the distance from the $j$-th historical entity as $d_j = \sum_{x\in F} \rho_x (\hat x, x_j)$, where $\hat x$ and $x_j$ denote the readout +# for feature $x$ for the entity being tested and the entity recorded in $j$-th column. +# The squared Mahalanobis distance directly takes in "rows", $\rho(\hat{x},x_{j})$. +# +# Next, we convert distances $d_j$ into probabilistic weights $w_j$. By default, we use a rescaled exponential function, i.e., +# we put $w_j = \exp(-\lambda d_j)$ with $\lambda=0.5$. Notably, $\lambda$'s value determines how belief is distributed across the historical entities. +# Larger values of $\lambda$ concentrate the belief tightly around the "closest" historical entities, while smaller values distribute more belief to more distant entities. +# +# Note that by choosing the squared Mahalanobis distance and the exponential function with a factor of $\lambda=0.5$, the resulting weight effectively equals the density of a [multivariate normal distribution](https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Density_function) fitted to the historical data. A similar assertion applies when we use the sum of "standardized" squared distances instead. However, in this latter case, we "enforce" the independence of the features. +# +# The proper choice of distance and similarity metrics depends on insight into the dataset at hand. Weights can then be used to construct +# weighted histograms or density estimators for the posterior distributions of interest, or to directly resample historical rows. + +# ![weights](assets/generative_weights.png) + +# ### Policy Search + +# Given these parts, searching for optimal experimental designs (actions arranged in an efficient way) depends on what our objective sense is. +# +# - The search may continue until the uncertainty about the posterior distribution of the target variable falls below a certain level. Our aim is to minimize the anticipated combined monetary cost and execution time of the search (considered as a "negative" reward). If all experiments are conducted without reaching below the required uncertainty level, or if the maximum number of experiments is exceeded, we penalize this scenario with an "infinitely negative" reward. +# - We may aim to minimize the expected uncertainty while being constrained by the combined costs of the experiment. +# - Alternatively, we could maximize the value of experimental evidence, adjusted for the incurred experimental costs. + +# Standard Markov decision process (MDP) algorithms can be used to solve this problem (offline learning) or construct the policy (online learning) for the sequential decision-making. + +# Our MDP's state space is finite-dimensional but generally continuous due to the allowance of continuous features, which complicates the problem and few algorithms specialize in this area. + +# We used the Double Progressive Widening Algorithm for this task as detailed in [A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation](https://arxiv.org/abs/1405.5498). + +# In a nutshell, the Double Progressive Widening (DPW) algorithm is designed for online learning in complex environments, particularly those known as Continuous Finite-dimensional Markov Decision Processes where the state space is continuous. The key idea behind DPW is to progressively expand the search tree during the Monte Carlo Tree Search (MCTS) process. The algorithm does so by dynamically and selectively adding states and actions to the tree based on defined heuristics. + +# In the context of online learning, this algorithm addresses the complexity and challenges of real-time decision-making in domains with a large or infinite number of potential actions. As information is gathered in actual runtime, the algorithm explores and exploits this information to make optimal or near-optimal decisions. In other words, DPW permits the learning process to adapt on-the-fly as more data is made available, making it an effective tool for the dynamic and uncertain nature of online environments. + +# In particular, at the core of the Double Progressive Widening (DPW) algorithm are several key components, including expansion, search, and rollout. + +# The search component is where the algorithm sifts through the search tree to determine the most promising actions or states to explore next. By using exploration-exploitation strategies, it can effectively balance its efforts between investigating previously successful actions and venturing into unexplored territories. + +# The expansion phase is where the algorithm grows the search tree by adding new nodes, representing new state-action pairs, to the tree. This is done based on a predefined rule that dictates when and how much the tree should be expanded. This allows the algorithm to methodically discover and consider new potential actions without becoming overwhelmed with choices. + +# Lastly, the rollout stage, also known as a simulation stage, is where the algorithm plays out a series of actions to the end of a game or scenario using a specific policy (like a random policy). The results of these rollouts are then used to update the estimates of the values of state-action pairs, increasing the accuracy of future decisions. + +# ![One iteration of the MCTS algorithm, taken from https://ieeexplore.ieee.org/document/6145622](assets/mcts.png) + +# In the above figure, nodes represent states of the decision process, while edges correspond to actions connecting these states. + +# A graphical summary of a single step of the overall search process to find the next best action, using our running example where a new entity +# has had $e_{1}$ performed out of 3 possible experiments is below: + +# ![search](assets/generative_search.png) + +# ## Synthetic Data Example with Continuous $y$ + +# We now present an example of finding cost-efficient generative designs on synthetic data using the CEEDesigns.jl package. +# +# First we load necessary packages. + +using CEEDesigns, CEEDesigns.GenerativeDesigns +using DataFrames +using ScientificTypes +import Distributions +using Copulas +using POMDPs, POMDPTools, MCTS +using Plots, StatsPlots, StatsBase + +# ### Synthetic Data + +# We use the "Friedman #3" method to make synthetic data for a regression problem from [scikit-learn](https://scikit-learn.org/stable/datasets/sample_generators.html#generators-for-regression). +# It considers $m=4$ features which predict a continuous $y$ via some nonlinear transformations. The marginal +# distributions of each feature are given by the scikit-learn documentation. We additionally use a Gaussian copula +# to induce a specified correlation structure on the features to illustrate the difference between Euclidean and Mahalanobis +# distance metrics. We sample $l=1000$ rows of the historical data. + +make_friedman3 = function (U, noise = 0, friedman3 = true) + size(U, 2) == 4 || error("input U must have 4 columns, has $(size(U,2))") + n = size(U, 1) + X = DataFrame(zeros(Float64, n, 4), :auto) + for i = 1:4 + X[:, i] .= U[:, i] + end + ϵ = noise > 0 ? rand(Distributions.Normal(0, noise), size(X, 1)) : 0 + if friedman3 + X.y = @. atan((X[:, 2] * X[:, 3] - 1 / (X[:, 2] * X[:, 4])) / X[:, 1]) + ϵ + else + ## Friedman #2 + X.y = @. (X[:, 1]^2 + (X[:, 2] * X[:, 3] - 1 / (X[:, 2] * X[:, 4]))^2)^0.5 + ϵ + end + return X +end + +ρ12, ρ13, ρ14, ρ23, ρ24, ρ34 = 0.8, 0.5, 0.3, 0.5, 0.25, 0.4 +Σ = [ + 1 ρ12 ρ13 ρ14 + ρ12 1 ρ23 ρ24 + ρ13 ρ23 1 ρ34 + ρ14 ρ24 ρ34 1 +] + +X1 = Distributions.Uniform(0, 100) +X2 = Distributions.Uniform(40 * π, 560 * π) +X3 = Distributions.Uniform(0, 1) +X4 = Distributions.Uniform(1, 11) + +C = GaussianCopula(Σ) +D = SklarDist(C, (X1, X2, X3, X4)) + +X = rand(D, 1000) + +data = make_friedman3(transpose(X), 0.01) + +data[1:10, :] + +# We can check that the empirical correlation is roughly the same as the specified theoretical values: + +cor(Matrix(data[:, Not(:y)])) + +# We ensure that our algorithms know that we have provided data of specified types. + +types = Dict( + :x1 => ScientificTypes.Continuous, + :x2 => ScientificTypes.Continuous, + :x3 => ScientificTypes.Continuous, + :x4 => ScientificTypes.Continuous, + :y => ScientificTypes.Continuous, +) +data = coerce(data, types); + +# We may plot each feature $x_{i} \in X = {x_{1},x_{2},x_{3},x_{4}}$ against $y$. + +p1 = scatter(data.x1, data.y; legend = false, xlab = "x1") +p2 = scatter(data.x2, data.y; legend = false, xlab = "x2") +p3 = scatter(data.x3, data.y; legend = false, xlab = "x3") +p4 = scatter(data.x4, data.y; legend = false, xlab = "x4") +plot(p1, p2, p3, p4; layout = (2, 2), legend = false) + +# ### Distance-based Similarity + +# Given historical data, a target variable $y$, and metric to quantify uncertainty around +# the posterior distribution on the target $q(y|e_{S})$, the function `DistanceBased` +# returns three functions needed by CEEDesigns.jl: +# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +# - `uncertainty`: this is a function of `evidence`, +# - `weights`: this represents a function of `evidence` that generates probability weights $w_j$ to each row in the historical data. +# +# By default, Euclidean distance is used as the distance metric. In the second +# call to `DistanceBased`, we instead use the squared Mahalanobis distance. +# It is possible to specify different distance metrics for each feature, see our +# [heart disease generative modeling](GenerativeDesigns.md) tutorial for more information. +# In both cases, the squared exponential function is used to convert distances +# to weights. + +(; sampler, uncertainty, weights) = DistanceBased( + data; + target = "y", + uncertainty = Variance(), + similarity = GenerativeDesigns.Exponential(; λ = 5), +); + +(sampler_mh, uncertainty_mh, weights_mh) = DistanceBased( + data; + target = "y", + uncertainty = Variance(), + similarity = GenerativeDesigns.Exponential(; λ = 5), + distance = SquaredMahalanobisDistance(; diagonal = 0), +); + +# We can look at the uncertainty in $y$ for a state where a single +# feature is "observed" at its mean value. +# As we considering only a single non-missing entry, note that the probabilistic weights assigned by the squared Mahalanobis distance +# are generally less "spread out." This is because the variant of the squared Mahalanobis distance, which we implemented for handling missing values, +# effectively multiplies the other quadratic distance by a factor greater than one. +# For more details, refer to page 285 of +# [Multivariate outlier detection in incomplete survey data: The epidemic algorithm and transformed rank correlations](https://www.jstor.org/stable/3559861). +# The interaction with uncertainties is more complex as the uncertainty depends on the values of the target variable that correspond to the rows in the historical data. + +data_uncertainties = + [i => uncertainty(Evidence(i => mean(data[:, i]))) for i in names(data)[1:4]] +sort!(data_uncertainties; by = x -> x[2], rev = true) + +data_uncertainties_mh = + [i => uncertainty_mh(Evidence(i => mean(data[:, i]))) for i in names(data)[1:4]] +sort!(data_uncertainties_mh; by = x -> x[2], rev = true) + +p1 = sticks( + eachindex(data_uncertainties), + [i[2] for i in data_uncertainties]; + xformatter = i -> data_uncertainties[Int(i)][1], + label = false, + title = "Uncertainty\n(Euclidean distance)", +) +p2 = sticks( + eachindex(data_uncertainties_mh), + [i[2] for i in data_uncertainties_mh]; + xformatter = i -> data_uncertainties_mh[Int(i)][1], + label = false, + title = "Uncertainty\n(Mahalanobis distance)", +) + +plot(p1, p2; layout = (1, 2), legend = false) + +# We can view the posterior distribution $q(y|e_{S})$ conditioned on a state (here arbitrarily set to $S = e_{3}$, giving evidence for $x_{3}$). + +evidence = Evidence("x3" => mean(data.x3)) +plot_weights = StatsBase.weights(weights(evidence)) +plot_weights_mh = StatsBase.weights(weights_mh(evidence)) + +p1 = Plots.histogram( + data.y; + weights = plot_weights, + legend = false, + ylabel = "Density", + title = "q(y|eₛ)\n(Euclidean distance)", +) +p2 = Plots.histogram( + data.y; + weights = plot_weights_mh, + legend = false, + ylabel = "Density", + title = "q(y|eₛ)\n(Mahalanobis distance)", +) + +plot(p1, p2; layout = (1, 2), legend = false) + +# Like static designs, generative designs need to be provided a `DataFrame` assigning to each experiment +# a tuple of monetary and time costs $(m_{e},t_{e})$, and what features each experiment provides observations of. +# We'll set up the experimental costs such that experiments which have less marginal uncertainty are more costly +# We finally add a very expensive "final" experiment which can directly observe the target variable. + +observables_experiments = Dict(["x$i" => "e$i" for i = 1:4]) +experiments_costs = Dict([ + observables_experiments[e[1]] => (i, i) => [e[1]] for + (i, e) in enumerate(data_uncertainties_mh) +]) + +experiments_costs["ey"] = (100, 100) => ["y"] + +experiments_costs_df = + DataFrame(; experiment = String[], time = Int[], cost = Int[], measurement = String[]); +push!( + experiments_costs_df, + [ + [ + e, + experiments_costs[e][1][1], + experiments_costs[e][1][2], + only(experiments_costs[e][2]), + ] for e in keys(experiments_costs) + ]..., +); +experiments_costs_df + +# ### Find Cost-efficient Experimental Designs + +# We can now find cost-efficient experimental designs for a new entity that has no measurements (`Evidence()`). +# Our objective sense is to minimize expected experimental combined cost while trying to reduce uncertainty to +# a threshold value. We examine 7 different threshold levels of uncertainty, evenly spaced between 0 and 1, inclusive. +# Additionally we set the `costs_tradeoff` such that equal weight is given to time and monetary cost when +# constructing the combined costs of experiments. +# +# Internally, for each choice of the uncertainty threshold, an instance of a Markov decision problem in [POMDPs.jl](https://github.com/JuliaPOMDP/POMDPs.jl) +# is created, and the `POMDPs.solve` is called on the problem. +# Afterwards, a number of simulations of the decision-making problem is run, all starting with the experimental `state`. +# +# Note that we use the Euclidean distance, due to somewhat faster runtime. + +n_thresholds = 7 +evidence = Evidence() +solver = GenerativeDesigns.DPWSolver(; n_iterations = 500, tree_in_info = true) +repetitions = 5 +mdp_options = (; + max_parallel = length(experiments_costs), + discount = 1.0, + costs_tradeoff = (0.5, 0.5), +) + +designs = efficient_designs( + experiments_costs; + sampler = sampler, + uncertainty = uncertainty, + thresholds = n_thresholds, + evidence = evidence, + solver = solver, + repetitions = repetitions, + mdp_options = mdp_options, +); + +# We plot the Pareto-efficient actions. + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") + +# ## Synthetic Data Example with Discrete $y$ + +# ### Synthetic Data + +# We use n-class classification problem generator from [scikit-learn](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html#sklearn.datasets.make_classification) +# We used parameters given below, for a total of $m=5$ features, with 4 of those informative and 1 redundant (linear combination of the other 4) feature. +# The $y$ has 2 classes, with some added noise. We again sample $l=1000$ rows of historical entities. +# +# The dataset can be approximately reproduced as below: + +## using PyCall +## sklearn = PyCall.pyimport_conda("sklearn", "scikit-learn") +## py""" +## import sklearn as sk +## from sklearn.datasets import make_classification +## """ +## X, y = py"make_classification(n_samples=1000,n_features=5,n_informative=4,n_redundant=1,n_repeated=0,n_classes=2,n_clusters_per_class=2,flip_y=0.1)" +## using DataFrames, CSV +## dat = DataFrame(X, :auto) +## dat.y = y +## CSV.write("./class.csv",dat) + +using CSV + +data = CSV.read("./data/class.csv", DataFrame) + +types = Dict( + :x1 => ScientificTypes.Continuous, + :x2 => ScientificTypes.Continuous, + :x3 => ScientificTypes.Continuous, + :x4 => ScientificTypes.Continuous, + :y => ScientificTypes.Multiclass, +) +data = coerce(data, types); + +# ### Distance-based Similarity + +# We now set up the distance based similarity functions. We use `Entropy` as our metric of uncertainty this time, +# which is more suitable for discrete $y$. + +(; sampler, uncertainty, weights) = DistanceBased( + data; + target = "y", + uncertainty = Entropy(), + similarity = GenerativeDesigns.Exponential(; λ = 5), +); + +# We may also look at the uncertainty from each marginal distribution of features. +# This is a bit nonsensical as the data generating function will create multimodal clusters +# so it will look artifically as if nothing is informative, but that is not the case. + +data_uncertainties = + [i => uncertainty(Evidence(i => mode(data[:, i]))) for i in names(data)[1:end-1]] +sort!(data_uncertainties; by = x -> x[2], rev = true) + +sticks( + eachindex(data_uncertainties), + [i[2] for i in data_uncertainties]; + xformatter = i -> data_uncertainties[Int(i)][1], + label = false, + ylab = "Uncertainty", +) + +# We can view the posterior distribution $q(y|e_{S})$ when we consider the state as evidence a single measurement +# of the first feature, set to the mean of that distribution. + +evidence = Evidence("x1" => mean(data.x1)) +plot_weights = StatsBase.weights(weights(evidence)) + +target_belief = countmap(data[!, "y"], plot_weights) +p = bar( + 0:1, + [target_belief[0], target_belief[1]]; + xrot = 40, + ylabel = "probability", + title = "unc: $(round(uncertainty(evidence), digits=1))", + kind = :bar, + legend = false, +); +xticks!(p, 0:1, ["0", "1"]); +p + +# Like previous examples, we'll set up the experimental costs such that experiments which have less marginal uncertainty +# are more costly, add a final very expensive experiment directly on the target variable. + +observables_experiments = Dict(["x$i" => "e$i" for i = 1:5]) +experiments_costs = Dict([ + observables_experiments[e[1]] => (i, i) => [e[1]] for + (i, e) in enumerate(sort(data_uncertainties; by = x -> x[2], rev = true)) +]) + +experiments_costs["ey"] = (100, 100) => ["y"] + +experiments_costs_df = + DataFrame(; experiment = String[], time = Int[], cost = Int[], measurement = String[]); +push!( + experiments_costs_df, + [ + [ + e, + experiments_costs[e][1][1], + experiments_costs[e][1][2], + only(experiments_costs[e][2]), + ] for e in keys(experiments_costs) + ]..., +); +experiments_costs_df + +# ### Find Cost-efficient Experimental Designs + +# We can now find sets of cost-efficient experimental designs for a new entity with no measurements (`Evidence()`). +# We use the same solver parameters as for the exaple with continuous $y$, and plot the resulting +# Pareto front. + +n_thresholds = 7 +evidence = Evidence() +solver = GenerativeDesigns.DPWSolver(; n_iterations = 500, tree_in_info = true) +repetitions = 5 +mdp_options = (; + max_parallel = length(experiments_costs), + discount = 1.0, + costs_tradeoff = (0.5, 0.5), +) + +designs = efficient_designs( + experiments_costs; + sampler = sampler, + uncertainty = uncertainty, + thresholds = n_thresholds, + evidence = evidence, + solver = solver, + repetitions = repetitions, + mdp_options = mdp_options, +); + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") diff --git a/tutorials/SimpleStatic.jl b/tutorials/SimpleStatic.jl new file mode 100644 index 0000000..09890e9 --- /dev/null +++ b/tutorials/SimpleStatic.jl @@ -0,0 +1,173 @@ +# # Static Experimental Designs + +# In this document we describe the theoretical background behind the tools in CEEDesigns.jl for producing optimal "static experimental designs," i.e., +# arrangements of experiments that exist along a Pareto-optimal tradeoff between cost and information gain. +# We also show an example with synthetic data. + +# ## Setting + +# Consider the following scenario. There exists a set of experiments, each of which, when performed, yields +# measurements on one or more observables (features). Each subset of observables (and therefore each subset of experiments) +# has some "information value," which is intentionally vaguely defined for generality, but for example, may be +# a loss function if that subset is used to train some machine learning model. It is generally the value of acquiring that information. +# Finally, each experiment has some monetary cost and execution time to perform the experiment, and +# the user has some known tradeoff between overall execution time and cost. +# +# CEEDesigns.jl provides tools to take these inputs and produce a set of optimal "arrangements" of experiments for each +# subset of experiments that form a Pareto front along the tradeoff between information gain and total combined cost +# (monetary and time). This allows informed decisions to be made, for example, regarding how to allocate scarce +# resources to a set of experiments that attain some acceptable level of information (or, conversely, reduce +# uncertainty below some level). +# +# The arrangements produced by the tools introduced in this tutorial are called "static" because they inherently +# assume that for each experiment, future data will deterministically yield the same information gain as the "historical" data did. +# This information gain from the "historical" data is quantified based on certain aggregate statistics. +# +# We can also consider "generative experimental designs," where the information gain is modeled as a random variable. This concept is detailed in another [tutorial](./SimpleGenerative.jl). +# +# This tutorial introduces the theoretical framework behind static experimental designs with synthetic data. +# For examples using real data, please see our other tutorials. + +# ## Theoretical Framework + +# ### Experiments + +# Let $E = \{ e_1, \ldots, e_n\}$ be a set of $n$ experiments (i.e., $|E|=n$). Each experiment $e \in E$ has an +# associated tuple $(m_{e},t_{e})$, giving the monetary cost and time duration required to perform experiment $e$. + +# ![experiments](assets/static_experiments.png) + +# Consider $P(E)$, the power set of experiments (i.e., every possible subset of experiments). Each subset of +# experiments $S\in P(E)$ has an associated value $v_{S}$, which is the value of the experiments contained in $S$. +# This may be given by the loss function associated with a prediction task where the information yielded from $S$ +# is used as predictor variables, or some other notion of information value. + +# ![experiments](assets/static_powerset.png) + +# ### Arrangements + +# If experiments within a subset $S$ can be performed simultaneously (in parallel), then each $S$ may be arranged +# optimally with respect to time. An arrangement $O_{S}$ of $S$ is a partition of the experiments in $S$ such that +# the size of each partition is not larger than the maximum number of experiments that may be done in parallel. +# +# Let $l$ be the number of partitions, and $o_{i}$ a partition in $O_{S}$. Then the arrangement is a surjection from $S$ +# onto $O_{S}$. If no experiments can be done in parallel, then $l=|S|$. If all experiments are done in parallel, then +# $l=1$. Other arrangements fall between these extremes. + +# ![experiments](assets/static_arrangement.png) + +# ### Optimal Arrangements + +# To find the optimal arrangement for each $S$ we need to know the cost of $O_{S}$. The monetary cost of $O_{S}$ is simply +# the sum of the costs of each experiment: $m_{O_{S}}=\sum_{e\in S} m_{e}$. +# The total time required is the sum of the maximum time *of each partition*. This is because while each partition in the +# arrangement is done in serial, experiments within partitions are done in parallel. That is, $t_{O_{S}}=\sum_{i=1}^{l} \text{max} \{ t_{e}: e \in o_{i}\}$. +# Given these costs and a parameter $\lambda$ which controls the tradeoff between monetary cost and time, the combined +# cost of an arrangement is: $\lambda m_{O_{S}} + (1-\lambda) t_{O_{S}}$. +# +# For instance, consider the experiments $S = \{e_{1},e_{2},e_{3},e_{4}\}$, with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$. +# If we conduct experiments $e_1$ through $e_4$ in sequence, this would correspond to an arrangement +# $O_{S} = (\{ e_1 \}, \{ e_2 \}, \{ e_3 \}, \{ e_4 \})$ with a total cost of $m_{O_{S}} = 4$ and $t_{O_{S}} = 10$. +# +# However, if we decide to conduct $e_1$ in parallel with $e_3$, and $e_2$ with $e_4$, we would obtain an arrangement +# $O_{S} = (\{ e_1, e_3 \}, \{ e_2, e_4 \})$ with a total cost of $m_{O_{S}} = 4$, and $t_{O_{S}} = 3 + 4 = 7$. +# +# Continuing our example and assuming a maximum of two parallel experiments, the optimal arrangement is to conduct +# $e_1$ in parallel with $e_2$, and $e_3$ with $e_4$. This results in an arrangement $O_{S} = (\{ e_1, e_2 \}, \{ e_3, e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 2 + 4 = 6$. +# +# In fact, it can be readily demonstrated that the optimal arrangement can be found by ordering the experiments in +# S in descending order according to their execution times. Consequently, the experiments are grouped sequentially +# into partitions whose size equals the maximum number of parallel experiments, except possibly for the final set, +# if the maximum number of parallel experiments does not divide $S$ evenly. + +# ## Synthetic Data Example + +# We now present an example of finding cost-efficient designs for synthetic data using the CEEDesigns.jl package. +# +# First we load necessary packages. + +using CEEDesigns, CEEDesigns.StaticDesigns +using Combinatorics: powerset +using DataFrames +using POMDPs, POMDPTools, MCTS + +# We consider a situation where there are 3 experiments, and we draw a value of their "loss function" +# or "entropy" from the uniform distribution on the unit interval for each. +# +# For each $S\in P(E)$, we simulate the information value ($v_{S}$) of $S$ as the product of +# the values for each individual experiment. +# Therefore, because smaller values are better, any subset containing multiple experiments is guaranteed to be +# more "valuable" than any component experiment. + +experiments = ["e1", "e2", "e3"]; +experiments_val = Dict([e => rand() for e in experiments]); + +experiments_evals = Dict(map(Set.(collect(powerset(experiments)))) do s + if length(s) > 0 + s => prod([experiments_val[i] for i in s]) + else + return s => 1.0 + end +end); + +# See our other tutorial on [heart disease triage](StaticDesigns.md) for an example of using CEEDesigns.jl's built-in +# compatability with machine learning models from `MLJ.jl` to evalute performance of experiments using +# predictive accuracy as information value. +# +# Next we set up the costs $(m_{e},t_{e})$ for each experiment. +# Better experiments are more costly, both in terms of time and monetary cost. We print +# the data frame showing each experiment and its associated costs. + +experiments_costs = Dict( + sort(collect(keys(experiments_val)); by = k -> experiments_val[k], rev = true) .=> + tuple.(1:3, 1:3), +); + +DataFrame(; + experiment = collect(keys(experiments_costs)), + time = getindex.(values(experiments_costs), 1), + cost = getindex.(values(experiments_costs), 2), +) + +# We can plot the experiments ordered by their information value. + +plot_evals( + experiments_evals; + f = x -> sort(collect(keys(x)); by = k -> x[k], rev = true), + ylabel = "loss", +) + +# We print the data frame showing each subset of experiments and its overall information value. + +df_values = DataFrame(; + S = collect.(collect(keys(experiments_evals))), + value = collect(values(experiments_evals)), +) + +sort(df_values, order(:value; rev = true)) + +# Now we are ready to find the subsets of experiments giving an optimal tradeoff between information +# value and combined cost. CEED exports a function `efficient_designs` +# which formulates the problem of finding optimal arrangements as a Markov Decision Process and solves +# optimal arrangements for each subset on the Pareto frontier. +# +# We set $\lambda=0.5$, the parameter controlling the relative weight given to monetary versus time costs +# with the tuple `tradeoff`. + +max_parallel = 2; +tradeoff = (0.5, 0.5); + +designs = efficient_designs( + experiments_costs, + experiments_evals; + max_parallel = max_parallel, + tradeoff = tradeoff, +); + +# Finally we may produce a plot of the set of cost-efficient experimental designs. The set of designs +# is plotted along a Pareto frontier giving tradeoff between informatio value (y-axis) and combined cost (x-axis). +# Note that because we set the maximum number of parallel experiments equal to 2, the efficient design for the complete set +# of experiments groups the experiments with long execution times together (see plot legend; each group within a partition is +# prefixed with a number). + +plot_front(designs; labels = make_labels(designs), ylabel = "loss") diff --git a/tutorials/StaticDesigns.jl b/tutorials/StaticDesigns.jl index 2a48a45..022cf0c 100644 --- a/tutorials/StaticDesigns.jl +++ b/tutorials/StaticDesigns.jl @@ -1,38 +1,20 @@ # # Heart Disease Triage # Consider a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. - -# ## Theoretical Framework - -# Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$. - -# For each subset $S \subseteq E$ of experiments, we denote by $v_S$ the value of information acquired from conducting experiments in $S$. - -# In the cost-sensitive setting of CEEDesigns, conducting an experiment $e$ incurs a cost $(m_e, t_e)$. Generally, this cost is specified in terms of monetary cost and execution time of the experiment. - -# To compute the cost associated with carrying out a set of experiments $S$, we first need to introduce the notion of an arrangement $o$ of the experiments $S$. An arrangement is modeled as a sequence of mutually disjoint subsets of $S$. In other words, $o = (o_1, \ldots, o_l)$ for a given $l\in\mathbb N$, where $\bigcup_{i=1}^l o_i = S$ and $o_i \cap o_j = \emptyset$ for each $1\leq i < j \leq l$. - -# Given a subset $S$ of experiments and their arrangement $o$, the total monetary cost and execution time of the experimental design is given as $m_o = \sum_{e\in S} m_e$ and $t_o = \sum_{i=1}^l \max \{ t_e : e\in o_i\}$, respectively. - -# For instance, consider the experiments $e_1,\, e_2,\, e_3$, and $e_4$ with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$. If we conduct experiments $e_1$ through $e_4$ in sequence, this would correspond to an arrangement $o = (\{ e_1 \}, \{ e_2 \}, \{ e_3 \}, \{ e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 10$. - -# However, if we decide to conduct $e_1$ in parallel with $e_3$, and $e_2$ with $e_4$, we would obtain an arrangement $o = (\{ e_1, e_3 \}, \{ e_2, e_4 \})$ with a total cost of $m_o = 4$, and $t_o = 3 + 4 = 7$. - -# Given the constraint on the maximum number of parallel experiments, we devise an arrangement $o$ of experiments $S$ such that, for a fixed tradeoff between monetary cost and execution time, the expected combined cost $c_{(o, \lambda)} = \lambda m_o + (1-\lambda) t_o$ is minimized (i.e., the execution time is minimized). - -# In fact, it can be readily demonstrated that the optimal arrangement can be found by ordering the experiments in set $S$ in descending order according to their execution times. Consequently, the experiments are grouped sequentially into sets whose size equals the maximum number of parallel experiments, except possibly for the final set. - -# Continuing our example and assuming a maximum of two parallel experiments, the optimal arrangement is to conduct $e_1$ in parallel with $e_2$, and $e_3$ with $e_4$. This results in an arrangement $o = (\{ e_1, e_2 \}, \{ e_3, e_4 \})$ with a total cost of $m_o = 4$ and $t_o = 2 + 4 = 6$. - -# Assuming the information values $v_S$ and optimized experimental costs $c_S$ for each subset $S \subseteq E$ of experiments, we then generate a set of cost-efficient experimental designs. +# For details on the theoretical background and notation, please see our tutorial on [static experimental designs](SimpleStatic.md), this tutorial +# is a concrete application of the tools described in that document. # ### Application to Predictive Modeling +# Let's generalize the example from [static experimental designs](SimpleStatic.md) to the case where we want to compute the information value $v_{S}$ as the predictive +# ability of a machine learning model which uses the measurements gained from experiments in $S$ to predict some $y$ of interest. +# +# Let's introduce some formal notation. # Consider a dataset of historical readouts over $m$ features $X = \{x_1, \ldots, x_m\}$, and let $y$ denote the target variable that we want to predict. - -# We assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. +# Assume that each experiment $e \in E$ yields readouts over a subset $X_e \subseteq X$ of features. # Then, for each subset $S \subseteq E$ of experiments, we may model the value of information acquired by conducting the experiments in $S$ as the accuracy of a predictive model that predicts the value of $y$ based on readouts over features in $X_S = \bigcup_{e\in S} X_e$. +# Then this accuracy is our information value $v_{S}$ of $S$. # ## Heart Disease Dataset @@ -44,7 +26,7 @@ data[1:10, :] # ## Predictive Accuracy -# The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +# The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. # We specify the experiments along with the associated features: @@ -96,7 +78,7 @@ model = classifier(; n_trees = 20, max_depth = 10) # ### Performance Evaluation -# We use `evaluate_experiments` from `CEEDesigns.StaticDesigns` to evaluate the predictive accuracy over subsets of experiments. We use `LogLoss` as a measure of accuracy. It is possible to pass additional keyword arguments, which will be passed to `MLJ.evaluate` (such as `measure`, shown below). +# We use `evaluate_experiments` from `CEEDesigns.StaticDesigns` to evaluate the predictive accuracy over subsets of experiments. We use `LogLoss` as a measure of accuracy. It is possible to provide additional keyword arguments, which will be passed to `MLJ.evaluate` (such as `measure`, shown below). using CEEDesigns, CEEDesigns.StaticDesigns @@ -112,8 +94,12 @@ perf_eval = evaluate_experiments( measure = LogLoss(), ) -# We plot performance measures evaluated for subsets of experiments. -plot_evals(perf_eval; ylabel = "logloss") +# We plot performance measures evaluated for subsets of experiments, sorted by performance measure. +plot_evals( + perf_eval; + f = x -> sort(collect(keys(x)); by = k -> x[k], rev = true), + ylabel = "logloss", +) # ## Cost-Efficient Designs @@ -134,7 +120,7 @@ plot_front(designs; labels = make_labels(designs), ylabel = "logloss") # ### Parallel Experiments -# We may exploit parallelism in the experimental arrangement. To that end, we first specify the monetary cost and execution time for each experiment, respectively. +# The previous example assumed that experiments had to be run sequentially. We can see how the optimal arrangement changes if we assume multiple experiments can be run in parallel. To that end, we first specify the monetary cost and execution time for each experiment, respectively. experiments_costs = Dict( ## experiment => (monetary cost, execution time) => features @@ -144,7 +130,7 @@ experiments_costs = Dict( "BloodSugar" => (20.0, 20.0) => ["FastingBS"], ) -# We set the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. +# We set the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time. In our case, we aim to minimize the execution time. # Below, we demonstrate the flexibility of `efficient_designs` as it can both evaluate the performance of experiments and generate efficient designs. Internally, `evaluate_experiments` is called first, followed by `efficient_designs`. Keyword arguments to the respective functions has to wrapped in `eval_options` and `arrangement_options` named tuples, respectively. diff --git a/tutorials/StaticDesignsFiltration.jl b/tutorials/StaticDesignsFiltration.jl index 258841e..0b57bf2 100644 --- a/tutorials/StaticDesignsFiltration.jl +++ b/tutorials/StaticDesignsFiltration.jl @@ -1,26 +1,27 @@ # # Heart Disease Triage With Early Droupout -# Consider again a situation where a group of patients is tested for a specific disease. It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. +# Consider again a situation where a group of patients is tested for a specific disease. +# It may be costly to conduct an experiment yielding the definitive answer. Instead, we want to utilize various proxy experiments that provide partial information about the presence of the disease. # Moreover, we may assume that for some patients, the evidence gathered from proxy experiments can be considered 'conclusive'. Effectively, some patients may not need any additional triage; they might be deemed healthy or require immediate commencement of treatment. By doing so, we could save significant resources. # ## Theoretical Framework -# Let us consider a set of $n$ experiments $E = \{ e_1, \ldots, e_n\}$. +# We take as a basis the setup and notation from the basic framework presented in the [static experimental designs tutorial](SimpleStatic.md). -# For each subset $S \subseteq E$ of experiments, we denote by $v_S$ the value of information acquired from conducting experiments in $S$. - -# Moreover, it can be assumed that a set of extrinsic decision-making rules is imposed on the experimental readouts. If the experimental evidence acquired for a given entity satisfies a specific criterion, that entity is then removed from the triage. However, other entities within the batch will generally continue in the experimental process. In general, the process of establishing such rules is largely dependent on the specific problem and requires comprehensive understanding of the subject area. +# We again have a set of experiments $E$, but now assume that a set of extrinsic decision-making rules is imposed on the experimental readouts. +# If the experimental evidence acquired for a given entity satisfies a specific criterion, that entity is then removed from the triage. +# However, other entities within the batch will generally continue in the experimental process. +# In general, the process of establishing such rules is largely dependent on the specific problem and requires comprehensive understanding of the subject area. # We denote the expected fraction of entities that remain in the triage after conducting a set $S$ of experiments as the filtration rate, $f_S$. In the context of disease triage, this can be interpreted as the fraction of patients for whom the experimental evidence does not provide a 'conclusive' result. -# In the cost-sensitive setting of CEEDesigns, conducting an experiment $e$ incurs a cost $(m_e, t_e)$. Generally, this cost is specified in terms of monetary cost and execution time of the experiment. - -# To compute the cost associated with carrying out a set of experiments $S$, we first need to introduce the notion of an arrangement $o$ of the experiments $S$. An arrangement is modeled as a sequence of mutually disjoint subsets of $S$. In other words, $o = (o_1, \ldots, o_l)$ for a given $l\in\mathbb N$, where $\bigcup_{i=1}^l o_i = S$ and $o_i \cap o_j = \emptyset$ for each $1\leq i < j \leq l$. +# As previously, each experiment $e$ incurs a cost $(m_e, t_e)$. Again, we let $O_{S}$ denote an arrangement of experiments in $S$. -# Given a subset $S$ of experiments and their arrangement $o$, the total (discounted) monetary cost and execution time of the experimental design is given as $m_o = \sum{i=1}^l r_{S_{i-1}}\sum_{e\in o_i} m_e$ and $t_o = \sum_{i=1}^l \max \{ t_e : e\in o_i\}$, respectively. Importantly, the factor $r_{S_{i-1}}$ models the fact that a set of entities may have dropped out in the previous experiments, hence saving the resources on running the subsequent experiments. +# Given a subset $S$ of experiments and their arrangement $O_{S}$, the total (discounted) monetary cost and execution time of the experimental design is given as $m_o = \sum_{i=1}^{l} r_{S_{i-1}}\sum_{e\in o_i} m_e$ and $t_o = \sum_{i=1}^{l} \max \{ t_e : e\in o_i\}$, respectively. +# Importantly, the new factor $r_{S_{i-1}}$ models the fact that a set of entities may have dropped out in the previous experiments, hence saving the resources on running the subsequent experiments. -# We note that these computations are based on the assumption that monetary cost is associated with the analysis of a single experimental entity, such as a patient. Therefore, the total monetary cost obtained for a specific arrangement is effectively the ['expected'](https://en.wikipedia.org/wiki/Expected_value) monetary cost, adjusted for a single entity. Conversely, we suppose that all entities can be concurrently examined in a specific experiment. As such, the total execution time is equivalent to the longest time until all experiments within an arrangement are finished or all entities have been eliminated (which ocurrs when the filtration rate the experiments conducted so far is zero). Importantly, this distinctly differs from calculating the 'expected lifespan' of an entity in the triage. +# We note that these computations are based on the assumption that monetary cost is associated with the analysis of a single experimental entity, such as a patient. Therefore, the total monetary cost obtained for a specific arrangement is effectively the ["expected"](https://en.wikipedia.org/wiki/Expected_value) monetary cost, adjusted for a single entity. Conversely, we suppose that all entities can be concurrently examined in a specific experiment. As such, the total execution time is equivalent to the longest time until all experiments within an arrangement are finished or all entities have been eliminated (which ocurrs when the filtration rate the experiments conducted so far is zero). Importantly, this distinctly differs from calculating the 'expected lifespan' of an entity in the triage. # For instance, consider the experiments $e_1,\, e_2,\, e_3$, and $e_4$ with associated costs $(1, 1)$, $(1, 3)$, $(1, 2)$, and $(1, 4)$, and filtration rates $0.9,\,0.8,\,0.7$, and $0.6$. For subsets of experiments, we simply assume that the filtration rates multiply, thereby treating the experiments as independent binary discriminators. In other words, the filtration rate for a set $S=\{ e_1, e_3 \}$ would equal $f_S = 0.9 * 0.7 = 0.63$. @@ -42,7 +43,7 @@ # - _actions_ are subsets of experiments which have not been conducted yet; the size of these subsets is restricted by the maximum number of parallel experiments; # - _reward_ is a combined monetary cost and execution time, discounted by the filtration rate of previously conducted experiments. -# Provided we know the information values $v_S$, filtration rates $r_S$, and experimental costs $c_S$ for each subset $S \subseteq E$, we find a collection of Pareto-efficient experimental designs that balance both the implied value of information and the experimental cost. +# Provided we know the information values $v_S$, filtration rates $r_S$, and experimental costs $c_S$ for each set of experiments $S$, we find a collection of Pareto-efficient experimental designs that balance both the implied value of information and the experimental cost. # ## Heart Disease Dataset @@ -90,7 +91,7 @@ data_binary[1:10, :] # In this scenario, we model the value of information $v_S$ acquired by conducting a set of experiments as the ratio of patients for whom the results across the experiments in $S$ were 'inconclusive', i.e., $|\cap_{e\in S}\{ \text{patient} : \text{inconclusive in } e \}| / |\text{patients}|$. Essentially, the very same measure is used here to estimate the filtration rate. -# The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. +# The CEEDesigns.jl package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. # We specify the experiments along with the associated features: diff --git a/tutorials/active_sampling.jl b/tutorials/active_sampling.jl new file mode 100644 index 0000000..46a38a1 --- /dev/null +++ b/tutorials/active_sampling.jl @@ -0,0 +1,277 @@ +# # Active Sampling Capability in Generative Designs + +# ## Background on Active Sampling + +# In multi-objective optimization (MOO), particularly in the context of active learning and reinforcement learning (RL), conditional sampling plays a critical role in achieving optimized outcomes that align with specific desirable criteria. The essence of conditional sampling in this context is to direct the optimization process not only towards the global objectives but also to adhere to additional, domain-specific constraints or features. This approach is crucial for several reasons: + +# ### Balancing Competing Objectives + +# 1. **Complexity of Multi-Objective Scenarios**: MOO often involves balancing several competing objectives, such as minimizing cost while maximizing performance. Conditional sampling allows for the integration of additional constraints or preferences, ensuring that the optimization does not overly favor one objective at the expense of others. + +# 2. **Navigating Trade-offs**: In MOO, trade-offs are inevitable. Conditional sampling helps navigate these trade-offs more effectively by providing a mechanism to prioritize or weigh different objectives based on additional conditions or context-specific requirements. + +# ### Enhancing Domain-Specific Relevance + +# 3. **Domain-Specific Constraints**: Many optimization problems have domain-specific constraints or desirable features that are not explicitly part of the primary objectives. For example, in drug design, beyond just optimizing for efficacy and safety, one might need to consider factors like solubility or synthesizability. Conditional sampling ensures that solutions not only meet the primary objectives but also align with these additional practical considerations. + +# 4. **Targeted Exploration**: Active learning and RL involve exploration of the solution space. Conditional sampling targets this exploration more effectively, focusing on regions of the solution space that are not only optimal in terms of the primary objectives but also relevant to the additional conditions or features. + +# ### Real-World Example: Drug Design + +# Consider the task of drug design, where the primary objectives might be to maximize efficacy against a target protein and minimize toxic side effects. However, suppose we also want the drug candidates to be easily synthesizable. This is a crucial feature for practical manufacturing but is not directly captured by the primary objectives of efficacy and toxicity. + +# In this scenario, conditional sampling becomes essential. By conditioning the sampling process on the synthesizability of the compounds, the optimization algorithm can explore regions of the solution space that not only have high efficacy and low toxicity but also are realistically manufacturable. Without this conditional approach, the algorithm might propose highly effective and safe drug candidates that are, however, chemically impractical to synthesize. + +# ### Summary + +# In summary, conditional sampling in MOO, especially in contexts involving active learning or RL, is crucial for ensuring that the optimized solutions are not only theoretically optimal with respect to the primary objectives but are also practically viable and relevant when additional domain-specific features or constraints are considered. This approach enhances the applicability and effectiveness of the optimization results in real-world scenarios. + +# For more information, refer to the following articles: + +# [Evolutionary Multiobjective Optimization via Efficient Sampling-Based Strategies](https://link.springer.com/article/10.1007/s40747-023-00990-z). +# [Sample-Efficient Multi-Objective Learning via Generalized Policy Improvement Prioritization](https://arxiv.org/abs/2301.07784). +# [Conditional gradient method for multiobjective optimization](https://link.springer.com/article/10.1007/s10589-020-00260-5) +# [A practical guide to multi-objective reinforcement learning and planning](https://link.springer.com/article/10.1007/s10458-022-09552-y) + +# ## Application to Generative Designs + +# `desirable_range` and `importance_weights` in DistanceBased.jl code serve different purposes, although they both influence the sampling process. + +# ## Desirable Range Constraints + +# The `desirable_range` is used to apply a filter to the data based on specified ranges for certain columns. The code snippet you provided checks if the values in a column fall within a specified range and creates a boolean array (`within_range`) that is true for rows that meet the criteria and false for those that don't. This boolean array is then used to element-wise multiply the similarities array, effectively setting the similarity to zero for rows that don't fall within the range. This is a form of hard constraint that excludes certain data points from being sampled. + +# ## Target Constraints + +# The `importance_weights`, on the other hand, apply a softer form of constraint by adjusting the weights of the data points based on certain conditions. Instead of excluding data points that don't meet the criteria, `importance_weights` increase the weight of those that do, making them more likely to be sampled, but still allowing for the possibility of sampling those that don't meet the criteria. + +# ## Comparison and Necessity + +# The key difference is that `desirable_range` acts as a binary filter (include or exclude), while `importance_weights` adjusts the probability of inclusion without necessarily excluding any data points. + +# Whether to increase the weight for elements within the range or simply filter them out depends on the specific requirements of the problem you're trying to solve: + +# - Filtering (Hard Constraint): If you are certain that data points outside the desirable range cannot possibly be relevant to your analysis or model, you might choose to filter them out completely. + +# - Weighting (Soft Constraint): If data points outside the desirable range could still be relevant, but you want to focus more on those within the range, then applying a weight is more appropriate. This allows for a more nuanced sampling that can account for uncertainty or the potential relevance of all data points. + +# In summary, whether to use filtering or weighting depends on the context and the desired strictness of the constraints. If strict exclusion is not necessary and you want to retain all data points with adjusted importance, then weighting is the appropriate choice. If certain data points should not be considered at all, filtering is the way to go. + +using CSV, DataFrames +data = CSV.File("data/heart_disease.csv") |> DataFrame +data[1:10, :] + +# We provide appropriate scientific types of the features. + +using ScientificTypes + +types = Dict( + :MaxHR => Continuous, + :Cholesterol => Continuous, + :ChestPainType => Multiclass, + :Oldpeak => Continuous, + :HeartDisease => Multiclass, + :Age => Continuous, + :ST_Slope => Multiclass, + :RestingECG => Multiclass, + :RestingBP => Continuous, + :Sex => Multiclass, + :FastingBS => Continuous, + :ExerciseAngina => Multiclass, +) +data = coerce(data, types); + +# ## Generative Model for Outcomes Sampling + +using CEED, CEED.GenerativeDesigns + +# As previously discussed, we provide a dataset of historical records, the target variable, along with an information-theoretic measure to quantify the uncertainty about the target variable. + +# In what follows, we obtain three functions: +# - `sampler`: this is a function of `(evidence, features, rng)`, in which `evidence` denotes the current experimental evidence, `features` represent the set of features we want to sample from, and `rng` is a random number generator; +# - `uncertainty`: this is a function of `evidence`, +# - `weights`: this represents a function of `evidence` that distributes probabilistic weights across the rows in the dataset. + +# Note that internally, a state of the decision process is represented as a tuple `(evidence, costs)`. + +# ## Active Sampling +# ### Desirable Range +# We might want to focus on data points that fall within one standard deviation from the mean for continuous variables. For example, for `:Age` with a mean of approximately 53.5, we could define a range around this mean. +mean_age = 53.5 +std_age = 9 # hypothetical standard deviation +desirable_range = Dict("Age" => (mean_age - std_age, mean_age + std_age)) + +# ### Target Constraints +# For binary variables like `:HeartDisease`, we might want to increase the weight of samples with the disease present to balance the dataset if it's imbalanced. +importance_weights = Dict("HeartDisease" => x -> any(x .== 1) ? 1.5 : 1.0) + +(; sampler, uncertainty, weights) = DistanceBased( + data, + "HeartDisease", + Entropy(), + Exponential(; λ = 5); + desirable_range = desirable_range, + importance_weights = importance_weights, +); + +# The CEEDesigns package offers an additional flexibility by allowing an experiment to yield readouts over multiple features at the same time. In our scenario, we can consider the features `RestingECG`, `Oldpeak`, `ST_Slope`, and `MaxHR` to be obtained from a single experiment `ECG`. + +# We specify the experiments along with the associated features: + +experiments = Dict( + ## experiment => features + "BloodPressure" => 1.0 => ["RestingBP"], + "ECG" => 5.0 => ["RestingECG", "Oldpeak", "ST_Slope", "MaxHR"], + "BloodCholesterol" => 20.0 => ["Cholesterol"], + "BloodSugar" => 20.0 => ["FastingBS"], + "HeartDisease" => 100.0, +) + +# Let us inspect the distribution of belief for the following experimental evidence: + +evidence = Evidence("Age" => 55, "Sex" => "M") +# +using StatsBase: countmap +using Plots +# +target_belief = countmap(data[!, "HeartDisease"], weights(evidence)) +p = bar( + 0:1, + [target_belief[0], target_belief[1]]; + xrot = 40, + ylabel = "probability", + color = :teal, + title = "unc: $(round(uncertainty(evidence), digits=1))", + kind = :bar, + legend = false, +); +xticks!(p, 0:1, ["no disease", "disease"]); +p + +# Let us next add an outcome of blood pressure measurement: + +evidence_with_bp = merge(evidence, Dict("RestingBP" => 190)) + +target_belief = countmap(data[!, "HeartDisease"], weights(evidence_with_bp)) +p = bar( + 0:1, + [target_belief[0], target_belief[1]]; + xrot = 40, + ylabel = "probability", + color = :teal, + title = "unc: $(round(uncertainty(evidence_with_bp), digits=2))", + kind = :bar, + legend = false, +); +xticks!(p, 0:1, ["no disease", "disease"]); +p + +# ## Cost-Efficient Experimental Designs for Uncertainty Reduction + +# In this experimental setup, our objective is to minimize the expected experimental cost while ensuring the uncertainty remains below a specified threshold. + +# We use the provided function `efficient_designs` to construct the set of cost-efficient experimental designs for various levels of uncertainty threshold. In the following example, we generate 6 thresholds spaces evenly between 0 and 1, inclusive. + +# Internally, for each choice of the uncertainty threshold, an instance of a Markov decision problem in [POMDPs.jl](https://github.com/JuliaPOMDP/POMDPs.jl) is created, and the `POMDPs.solve` is called on the problem. Afterwards, a number of simulations of the decision-making problem is run, all starting with the experimental `state`. + +## set seed for reproducibility +using Random: seed! +seed!(1) +# +evidence = Evidence("Age" => 35, "Sex" => "M") +# +## use less number of iterations to speed up build process +solver = GenerativeDesigns.DPWSolver(; + n_iterations = 100,#20_000 + exploration_constant = 5.0, + tree_in_info = true, +) +designs = efficient_designs( + experiments, + sampler, + uncertainty, + 6, + evidence; + solver, + mdp_options = (; max_parallel = 1), + repetitions = 5, +); + +# We plot the Pareto-efficient actions: + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") + +# We render the search tree for the second design, sorted in descending order based on the uncertainty threshold: + +using D3Trees +d3tree = D3Tree(designs[2][2].tree; init_expand = 2) + +# ### Parallel Experiments + +# We may exploit parallelism in the experimental arrangement. To that end, we first specify the monetary cost and execution time for each experiment, respectively. + +experiments = Dict( + ## experiment => (monetary cost, execution time) => features + "BloodPressure" => (1.0, 1.0) => ["RestingBP"], + "ECG" => (5.0, 5.0) => ["RestingECG", "Oldpeak", "ST_Slope", "MaxHR"], + "BloodCholesterol" => (20.0, 20.0) => ["Cholesterol"], + "BloodSugar" => (20.0, 20.0) => ["FastingBS"], + "HeartDisease" => (100.0, 100.0), +) + +# We have to provide the maximum number of concurrent experiments. Additionally, we can specify the tradeoff between monetary cost and execution time - in our case, we aim to minimize the execution time. + +## minimize time, two concurrent experiments at maximum +seed!(1) +## use less number of iterations to speed up build process +solver = GenerativeDesigns.DPWSolver(; + n_iterations = 100, + exploration_constant = 5.0, + tree_in_info = true, +) +designs = efficient_designs( + experiments, + sampler, + uncertainty, + 6, + evidence; + solver, + mdp_options = (; max_parallel = 2, costs_tradeoff = (0, 1.0)), + repetitions = 5, +); + +# We plot the Pareto-efficient actions: + +plot_front(designs; labels = make_labels(designs), ylabel = "% uncertainty") + +# ## Efficient Value Experimental Designs + +# In this experimental setup, we aim to maximize the value of experimental evidence, adjusted for the incurred experimental costs. + +# For this purpose, we need to specify a function that quantifies the 'value' of decision-process making state, modeled as a tuple of experimental evidence and costs. + +value = function (evidence, (monetary_cost, execution_time)) + return (1 - uncertainty(evidence)) - (0.005 * sum(monetary_cost)) +end + +# Considering a discount factor $\lambda$, the total reward associated with the experimental state in an $n$-step decision process is given by $r = r_1 + \sum_{i=2}^n \lambda^{i-1} (r_i - r_{i-1})$, where $r_i$ is the value associated with the $i$-th state. + +# In the following example, we also limit the maximum rollout horizon to 4. +# +seed!(1) +## use less number of iterations to speed up build process +solver = + GenerativeDesigns.DPWSolver(; n_iterations = 20_000, depth = 4, tree_in_info = true) +design = efficient_value( + experiments, + sampler, + value, + evidence; + solver, + repetitions = 5, + mdp_options = (; discount = 0.8), +); +# +design[1] # optimized cost-adjusted value +# +d3tree = D3Tree(design[2].tree; init_expand = 2) diff --git a/tutorials/assets/generative_actions.png b/tutorials/assets/generative_actions.png new file mode 100644 index 0000000..11b779c Binary files /dev/null and b/tutorials/assets/generative_actions.png differ diff --git a/tutorials/assets/generative_historical.png b/tutorials/assets/generative_historical.png new file mode 100644 index 0000000..eadbb6e Binary files /dev/null and b/tutorials/assets/generative_historical.png differ diff --git a/tutorials/assets/generative_search.png b/tutorials/assets/generative_search.png new file mode 100644 index 0000000..a99683f Binary files /dev/null and b/tutorials/assets/generative_search.png differ diff --git a/tutorials/assets/generative_state.png b/tutorials/assets/generative_state.png new file mode 100644 index 0000000..5e9dfeb Binary files /dev/null and b/tutorials/assets/generative_state.png differ diff --git a/tutorials/assets/generative_weights.png b/tutorials/assets/generative_weights.png new file mode 100644 index 0000000..7f9c21f Binary files /dev/null and b/tutorials/assets/generative_weights.png differ diff --git a/tutorials/assets/static_arrangement.png b/tutorials/assets/static_arrangement.png new file mode 100644 index 0000000..d9a954f Binary files /dev/null and b/tutorials/assets/static_arrangement.png differ diff --git a/tutorials/assets/static_experiments.png b/tutorials/assets/static_experiments.png new file mode 100644 index 0000000..9f63d7a Binary files /dev/null and b/tutorials/assets/static_experiments.png differ diff --git a/tutorials/assets/static_powerset.png b/tutorials/assets/static_powerset.png new file mode 100644 index 0000000..d514bcd Binary files /dev/null and b/tutorials/assets/static_powerset.png differ diff --git a/tutorials/data/class.csv b/tutorials/data/class.csv new file mode 100644 index 0000000..dda6e44 --- /dev/null +++ b/tutorials/data/class.csv @@ -0,0 +1,1001 @@ +x1,x2,x3,x4,x5,y +-0.39988512043361646,1.3337073852278094,1.1399557169899495,1.1656889925296003,0.1028643166223937,0 +0.11604658695262537,0.12370814601071312,1.314537694005196,1.347981083362914,-1.4885043346989004,0 +0.17817098696507017,-0.4940225050295366,0.7101473442842716,0.6751970915184036,-1.4634551594417935,0 +-1.1712824011295664,0.04258831266945029,1.62025299893532,1.71087271300213,-1.449551858671816,0 +0.9131479655930531,-1.457986249717023,0.8826851445584001,1.1959448704064282,-2.504196488731152,0 +-0.3471865322162061,2.6476894973159713,1.1821266682914224,-0.37015731553094144,-0.387102983473691,1 +0.8146480451580892,0.5407223388443838,1.9284968461160732,1.9477565227431675,-2.072595692705426,0 +-1.863027358704509,-2.064648810239415,-1.2006661227151492,0.36567595689210564,1.7511483753093189,1 +-2.188227128112658,1.5366361127776336,3.5424988094574195,4.394714086659788,-1.1269768750094369,1 +0.25251658133550636,0.3031942523561524,0.9371226649945322,1.5767164463013135,-0.224483264186758,1 +0.002839304263283937,2.6693047445091818,0.6739775688676186,0.38393563984935275,1.5297046259184675,1 +-0.15765068614618127,-0.584960313389373,1.6174417357967157,1.7897073987828693,-2.3252752016010594,0 +-1.588671276196846,-0.7253429104252509,0.12871718758431017,1.6980802479231558,1.3763782270136524,1 +-0.5539795119187028,0.8641465276170883,1.1831377666655136,1.203119843349317,-0.3753730029975938,0 +-3.019655172797125,-2.353284160287712,-0.756619475585828,1.9363809339481173,2.5465320170459496,1 +-2.2302994428484015,3.698457373872727,1.702969084719038,-0.9199008126963157,-0.5191667065979122,0 +1.4219932437113503,-1.3852574357495575,0.15006304416893193,1.192437738256401,-0.9015312471067121,1 +0.33528525842650936,-1.0583267029426657,1.2005676128416884,3.8794998410712656,0.30601064761181784,1 +-0.99460690730578,1.4563728592217473,0.9150874112802068,0.4570560762560669,0.1680623705018709,0 +0.23140506706972497,1.551542933942709,-0.30803453985981544,-1.3114388637286236,0.7500362837752963,0 +0.09571378722300405,0.8766197248160615,0.5570712567023978,0.7246971078686033,0.34876599548223997,0 +-1.3126003249409937,0.38281298783407613,1.7629191942417348,1.686551435699979,-1.4197734111591906,0 +0.44218935006583726,0.8412828961933221,1.5222756065494971,1.420435629528647,-1.2820778759678264,0 +-0.5326566627094349,-0.9604334112597348,-0.8816563838958104,-0.15276630250099577,1.0987107055733827,1 +0.7302680218871171,0.9906695152209604,2.8332698250879425,2.9688234397952176,-2.5736099471570455,0 +-1.0392284305363786,0.7760894532153282,1.9320306308421653,2.0555619218077528,-1.1042882754484817,0 +-1.9026497433532115,-2.740984092566441,-1.8582830859721775,0.1439942741919047,2.37217632728401,1 +-1.7484369756318965,0.6450491389743567,1.8716757632544052,2.888407118614435,0.05982366085196411,1 +1.3590038150097192,0.3841468074075277,0.32990463800105696,-0.025743076177584312,-0.866611506814594,0 +-1.6479385244824136,1.97615375796191,0.942025099668057,0.5112315792164399,0.9052476952381469,1 +-0.5659567834195822,0.5410611803100303,1.5752955530716046,2.1174936224627854,-0.600558330585039,1 +-0.20189033568009584,0.10603824702787712,-1.892749279969117,-1.724799677479691,2.6777373807625984,1 +0.5845804037322986,-0.06893575100251259,0.8299095969712593,1.8216223685944646,-0.1897873243294168,1 +-2.3161910011712736,3.3421119396836843,2.1972670146871827,0.40907748426327806,-0.5339294539843672,0 +0.8104938049569008,-0.8774252601102237,-0.3540597257029062,0.03218458727704887,-0.2923174684756149,1 +0.9794765452898768,-1.3523156943512733,0.5737997850946803,1.92905005411041,-0.894629770316831,0 +-1.342998410449443,-2.073374717926688,0.03524748391112986,3.157990673440001,1.7678401006007478,1 +1.1454017623671824,0.15623894759253365,-0.47106695228735274,-0.07096741606110513,0.7900960055960877,1 +1.1230061111887255,1.8758659685891437,1.1688942064882903,0.8518335796669269,-0.2774673385828619,0 +-0.47399281769159063,0.3921123029107083,1.0336859393929647,0.5074139337344216,-1.2938429180683717,0 +0.39695144216302447,2.24500987841873,1.8358856567485207,1.9086361339525242,-0.051723598267559234,1 +-1.252657098940631,-2.2371993264915115,-0.8809433452083446,0.7864673676346161,1.0930513353458666,1 +1.227777218867132,0.5678791678041634,-1.3147754582220097,-1.1044983578133225,1.9996654624928685,1 +2.7177144448080233,3.279216220230244,1.0107162303305897,0.32047860051265065,0.3771511240585472,0 +0.34170742335316306,0.6574516825778183,1.5767034195334615,1.6443118646895647,-1.312720932913606,0 +-1.0273458541165177,3.139604040181855,0.5284307676359961,-1.3334780415299767,0.7940112516017646,0 +0.5000972392852311,1.2600194132774851,3.0589695375657433,3.1926302672835916,-2.504985636583183,0 +-1.5768471501965986,-3.101460742738223,-0.7210631890090313,2.644400635078993,2.0095535904774593,1 +-0.0687091133622284,1.1099363576465653,1.2278473373223706,1.5303631617872808,-0.03540755496737957,1 +1.694041836780091,1.3887269463273704,0.9545909077940009,0.5762107052063541,-0.7623208699884604,0 +0.17565053324161384,-1.8795306049358018,-0.7912307538040068,0.9100880580025064,0.8998991297086507,1 +0.974788666041509,-0.5259535966636866,-0.13528107648643845,-0.1621088101627579,-0.7180233235496399,1 +-1.2799898121065132,2.069540357367523,1.224501416319961,-0.02543480438022061,-0.3726961042499115,0 +-0.8160826168926728,-1.0682308984810533,-0.2599112999353876,0.5045564713834325,0.3632199808768607,1 +-1.966611004874168,5.143180093091348,1.5311724641757343,-1.368784808611951,0.7459099261638829,0 +0.7084421694524579,2.37660565267916,0.6903810953296443,0.13660478370532292,0.6890783285652184,1 +-1.042375598601283,1.0710852812053886,-0.3926509757574178,0.14210418355659382,2.4932493913690426,1 +1.2083511796741444,-1.6002672800658504,-0.9273725883632992,0.060702624189509,0.21475291526726004,1 +-0.5349426563093,-1.919233366573267,0.7862269265107295,1.271114692577681,-2.1736276053700023,0 +-2.0855064200523437,-1.286326457429396,-0.4501376037130722,1.7089853679249796,2.3393138066685957,0 +0.023181263887859016,0.26710729989978255,-0.24918411387759687,0.07418818332866017,0.9213559584808393,1 +-0.11262807820639742,1.9282367376968323,0.12211674809090606,-0.04524717905409026,1.6367544617469227,1 +-1.562266414044001,3.91140520095054,2.259827841834646,1.4046779660119169,0.7346946021136926,1 +-4.428295465939227,1.4790391846360043,1.6009527495553155,2.3948416743938106,1.8810237029346508,1 +-1.2299188052232597,-3.168167515709092,-1.682860328407192,0.3018860641404201,1.4840862898811502,1 +0.6406256766743919,1.1992443480007198,1.8213617472705823,2.2202243097677634,-0.8048284300813728,1 +-0.5524768731809919,-1.3367373265020586,1.47945027633624,2.362863854966524,-1.9942264502089997,1 +-1.1001449704883777,-0.09100743297252478,1.2519731055094478,0.4646891869028046,-2.122943291445078,0 +3.02340362402904,-1.7265826012417689,1.915617047167827,5.518951575192933,-1.123575426541911,1 +0.5495796428297566,-2.7117422459451483,1.1315243392902423,3.2200323394406536,-1.9888898082974436,0 +0.6206020949277804,-0.021078667227930614,1.3312176941033347,1.4782771594850999,-1.6982065854048138,0 +-0.6383069610907426,-1.0231723798559047,-0.022244641924678565,1.1430843745850978,0.49903726143328453,1 +-1.602436550541821,2.2815144150032376,1.6988488486678965,-0.28180691224058774,-1.4378588704251127,0 +2.543880393273679,2.2636916545767933,2.254406352185062,1.827698094768528,-1.8159189566568443,0 +-0.9456358190090774,1.0727250490293492,1.089117177205299,1.7046145517840432,0.7360756700932674,1 +-2.9811845626549314,-0.3687736065870666,-0.25115782032436185,0.8121624506753856,2.1090210976833563,1 +-2.440418840792269,-2.3753997793684407,-1.5770626655484015,0.4162607926879637,2.564656269219452,1 +-0.966778564636996,-1.851613260642643,-1.0067051261601625,0.13061384287983646,0.9539798946779102,1 +-0.3310629620072062,0.9215548687347038,1.2111534754172626,1.0850436334684088,-0.5878063620298832,0 +-1.2999968705058151,0.8088230896893808,1.6297627085941842,1.0429458990187492,-1.3959641305232235,0 +0.8394967191170741,0.7262349553536211,3.1537076356699982,3.373339403490877,-3.1752935301163974,0 +-1.7908857605968178,1.962991490408586,1.7314129963548224,1.6432135637840317,0.3502205399270687,1 +-0.14668617091136427,1.3119507709575815,0.41341434643023267,0.024506807304796152,0.4295722816627525,0 +-0.04105530194044826,0.8745682041764067,0.4437974500929688,0.3377787156869606,0.23000896568607554,0 +0.4380154618528218,2.1632169614454417,2.1895079170200074,1.787313589024771,-1.1037320271478652,0 +0.1344092807008681,1.60141864414982,-0.30645978194555795,-0.501969318636718,1.7203261603367386,1 +-3.531215401596665,-3.6234502459925295,-1.3780734885059267,2.1054900049311076,3.0768351065421387,1 +-0.971076609785853,0.8018095667984867,1.304180230589901,0.9247011345107701,-0.8861363521048304,0 +-0.6762560123708579,1.7272392482678094,1.2059859529651245,0.5784361940199746,-0.21026189185262933,0 +-1.2128773938785697,1.141319253300938,2.141512130136142,1.449518242045872,-1.8346490890558997,0 +-0.6698391409513256,-0.6761049501968288,0.027968404001009306,1.2312152269518322,0.8377602400407796,1 +-1.8027067813783524,-2.7112839663166106,-1.7686886739636862,0.3300618579880855,2.364902567367835,1 +-1.0742081535420418,1.291694091996956,1.4732860133934313,1.0683013164183963,-0.595731790957813,0 +-2.7796706886187543,5.626749333286726,1.8624426378243188,-1.3185545711676787,0.7880570612175586,0 +0.7257848927633654,1.060579895499629,0.6906016934274193,0.9583113036697242,0.26844616365874074,1 +-2.37522766822483,-2.0451239287964667,-1.180481853099383,0.9666969445196764,2.557449004775004,1 +2.690019885753384,0.6775412615597949,2.6847603167114453,2.99840395470338,-3.166826566555012,0 +-1.4527241148341803,0.7968026108250906,2.236030456774712,2.1902061748677717,-1.5037450523331952,0 +1.4994989967196761,-1.0098105059361089,0.34466268120099847,0.5272309664379385,-1.7364317785011982,1 +0.6572318193277807,-0.06657889226576374,0.6566714533282095,0.6161774089445622,-1.1362727778301251,1 +2.5526070115695867,1.434105568343269,0.6120081943422102,0.4466609188622269,-0.3505430941887021,0 +-1.0102227607100855,3.8357361941069668,0.8541760269380676,-0.7615969382823575,1.358068519138766,0 +1.4968746279453529,2.1978874127509824,-0.24658097769279813,-0.8971842255940903,1.2866762789672306,0 +-1.2204179063074878,3.7559279914904526,0.7976902493252214,-0.8654861309628465,1.365573897360882,0 +-1.3622610294318305,-0.9652773047207448,2.3255240188450084,2.4333614751372403,-3.2408358149506107,0 +-1.2947104671925396,0.9207032503037553,1.64406253771807,1.515803670948587,-0.7981234866924097,0 +-0.8085505608806088,-0.1892958642687469,-0.3400889428238974,-0.6728241900217808,0.13168664331508773,1 +1.121501327537658,1.8062239412293561,1.7772074905372004,1.5696760721652059,-0.9713863321345013,0 +0.17342903105151497,-1.5638837150495446,1.5275223291005042,3.1513042076572413,-1.7089906077550148,0 +1.5313614732764695,0.18156574216197297,0.2561231404380712,0.3716760466853982,-0.5182040967414809,1 +0.21948162034841334,-2.933230222311794,-0.36554571593541785,3.0736681852182146,1.2212992318985685,1 +-2.861360456902147,-3.2299395182719293,-1.1379726327426374,2.062987743943878,2.6412381908510065,1 +-2.109413225945374,-3.3757824081111143,-0.8479795833621844,2.649459127728171,2.214036156046829,0 +-1.0530647535257225,-2.1294462941111147,0.6942059245675882,4.299072791114557,1.3380343909750676,1 +-2.3361131705204916,1.2954641994176272,2.4866719270583455,1.4367887996848725,-2.1203186325089938,0 +-1.3903924477941891,-0.753803126614216,1.949271226645867,2.272329554281728,-2.3214660221422356,0 +-1.5122537883904568,-2.3982689049770536,-1.9047277520058847,-0.8553376993292074,1.5923015121584692,1 +-0.8318832527987848,-4.105529463987493,-0.769813584835227,3.2052302641957864,1.4841873408083464,1 +-2.0152050241982553,0.7288340027981387,2.6667886668487975,2.1011355949749184,-2.48336527287508,0 +-2.3058656067775,-0.5741407341886278,0.12052574062527144,1.129960945445374,1.161944274485102,1 +-0.45460551697253593,-2.80532143251547,-0.7898107182456211,1.463011029253518,0.7888908515133407,1 +-2.2643448119278435,1.6531811584090434,2.767888112181035,3.4834579077327943,-0.18613777985865676,1 +-0.8674772145052044,0.48255432954590427,-0.30785438404770377,-1.062151352077517,0.32119222893882404,1 +-1.5773109254432363,1.339890715494202,1.0874775359442197,2.064653624178626,1.6160816100117374,1 +-2.88613063253072,-1.0434874909692404,-0.5363202674720657,1.3060120116425085,2.608021551445627,1 +0.03957915492683073,-0.21971374168436708,1.0415764893278059,1.641829165282114,-0.8485110205941212,0 +-1.0637078225626546,-0.9667530804038879,-0.5761125228040718,0.2643142252166729,1.0190276833369347,1 +-1.5595711052631271,-1.6772783128100204,-0.3146247767816769,1.364118146168146,1.0759692755933665,1 +-1.136478144714418,2.419814644058498,1.543779203507345,0.7408950554333074,0.03145343826710767,1 +-0.8205745041335408,-0.982053132593995,-1.4691048243451759,-0.6253774406949286,2.019753799360877,1 +-1.1857430705437462,-1.0623088029381738,-1.970574375811211,-2.5748608130410533,1.0815272704648806,1 +-1.379015157677912,1.1178061152315832,1.6845389211075799,0.946677547877089,-1.2931716001869022,1 +2.572429009456915,1.1142875159383154,2.1375852964114888,2.1719925785418566,-2.3267559907882616,0 +0.4028448243112941,0.9425994976380745,0.4769270905889601,-0.19779640288418232,-0.5175670266140191,1 +-1.4332927484257558,2.0120055446450653,1.337798930892697,0.6091843290501067,0.05635715673132058,0 +-0.14539644945775265,1.8655295512421206,2.562932040718172,2.281804355491441,-1.5305848226559344,0 +-1.717651649632793,-0.7375639036860908,0.32641837991317146,1.3327173769479161,0.5451251457059999,1 +-3.152258329639155,-0.19674316307229933,0.6347709775079505,1.7755376028507612,1.3388063153322785,1 +1.9092356577251914,1.9958364611985928,1.8881797725067877,1.7493260913261173,-1.105877625179031,1 +0.3179884580855621,-2.4048778247132736,1.0824509354712046,3.246135813038887,-1.4606929575100356,0 +-2.063139220279875,0.42769620153913235,-0.24178015079019016,-0.1424800505201429,1.5263504119959772,1 +-0.7403711983761396,-1.5927228992246443,-0.3340129318379125,1.6811796640410637,1.280195786159192,1 +-2.06040179119844,1.7007383565108016,2.222804508868918,1.2348747048880133,-1.4150697340673328,0 +1.3943913171483058,1.246451795287605,2.419991273046495,2.4129046680759547,-2.1907704067400604,0 +-1.4826136781476222,0.9230354839139653,1.9188512048171913,1.2368586024787063,-1.6791861944469106,0 +-2.6750435362836176,-1.8623069413686126,-1.5037151724206077,-0.2361262854389128,2.2686248812935266,1 +-1.9694635529084281,0.49552056984938875,2.9951594061267306,2.230685915584953,-3.353814028255917,0 +0.4611114255906521,2.5270872047282245,1.5419623179103352,1.437812203696176,0.37478072681730634,1 +0.36052150400120997,-1.5203229042214872,0.9951311330416465,3.735087347281806,0.15331349479883782,1 +-0.7452779705850647,-2.9258006287704643,2.3890331159170364,4.00591269392447,-3.8290342696432953,0 +-0.45656094698280936,-0.1483653837949197,-0.1646146694203282,0.5531206166146156,0.9964932712357841,1 +-1.235461117507949,-2.761577006228793,-1.1484067003556238,1.1244714956040265,1.5560704088307729,1 +-1.351484872683368,0.43701810779519235,1.45516973437466,2.3239244116089353,0.06543042383589914,1 +-1.7199946846736243,1.6339616676306794,2.1525345458300276,1.1871324935055285,-1.4853005914978221,1 +0.38608164752349083,-0.07957019482201977,2.778026482010281,3.0805066951898312,-3.2794698725818083,0 +-1.0587462275869144,-1.004076780455612,-0.13566076338713534,-0.28124489116693474,-0.6454735583870557,0 +0.6670108775735097,2.4671085051887003,0.1748656144057161,-0.2687193507166836,1.5465916374573307,1 +-1.2515715945591679,-1.6465656481572286,-0.36269109458383497,1.715179142795547,1.5018591616966361,1 +0.15050089846987857,1.2499037865346743,0.9018210361699831,0.3402308525011686,-0.5208161568550818,0 +-1.1496876451863487,-0.25442506678141297,-0.9836589185392473,-0.6434370120955206,1.7103449677072624,0 +-0.16271542793796578,-2.065667146577949,1.172153676818231,1.9705198524463863,-2.5728518868560215,0 +1.183138995519342,1.6743658240105193,0.83169169775719,0.23029764074905246,-0.3996441901014067,1 +-1.273794446946026,0.25517795300171797,0.33602828852720007,1.0483685959607287,1.0558780092903277,1 +-3.360157107831941,-0.04950293085677726,-0.34250692420913365,0.03438003030212955,1.9121676446377447,1 +-3.564583678249846,0.47444017103475644,-0.6360363245064411,-0.04501945278175179,3.1013527975513697,1 +-3.7784196196501045,0.2694647216635415,2.46953120938777,3.4654500265065886,-0.3910578388925614,1 +0.5637407316611003,2.624447540182201,1.6181928108933197,1.1186386025320474,-0.09087411739607654,0 +-1.8196171960825556,0.3012193942357726,-0.7290630642425491,-1.2913599843199592,1.1862631732325915,0 +0.8383080115370747,1.6300557123781565,2.104662598135927,1.938256910094327,-1.4088524213429134,0 +3.460292538043455,0.8811741641538542,1.8944403205289482,1.9113624447946593,-2.5796443673094167,1 +-0.3719561952346768,-1.6142180651774543,1.2372118948191038,1.666356517710417,-2.5364289779941496,0 +-0.021968570557738487,0.8209752200973923,0.5953278003654234,0.7911463808000105,0.3166430353087514,1 +-1.1334665139825209,1.2153442070374183,1.5894985996841997,2.583588076060166,0.7459596195678235,0 +-0.02737119086763795,0.31469908268337266,-0.6271327798146641,-1.26947493860989,0.38555325994272605,1 +-0.17159053229648003,1.4941483065567072,1.295940052640618,1.1944575434668279,-0.1440766767230286,1 +0.527995330559287,-0.07394530719157988,0.9302634630519093,1.6045994053752137,-0.6482773672304578,1 +-0.09754373362922242,-1.1747702929428674,1.101249174799772,3.87435896100766,0.5598618792238925,0 +-0.44411016849247675,-0.05669341065013067,0.01821745918643647,0.6781871885456677,0.7966120768526541,1 +-4.180095994293889,-0.7519047170165374,-1.2142746936556486,-1.1130945805825228,2.2474823316624355,1 +2.0024243881555295,1.6732656324090316,2.4429970751568937,2.4034966388920793,-2.030768935507846,1 +0.3513510294616805,0.7891161198688877,1.9736342878049118,1.9384680515887018,-1.7834253625605383,0 +0.9262855120214887,2.6726619219748464,1.5239487897620083,0.9283315829451703,-0.15448951882038642,0 +-1.6511470416453706,2.92370020207699,1.424848031471328,-0.5591266832127899,-0.44576778019144137,0 +-0.6722954724120327,1.4278133661393675,1.1521018096303366,0.14141151159101517,-0.8676555485197377,0 +-2.665141848123402,2.0562133105704077,1.9143062769862578,2.3980684537260855,1.1427406009017274,1 +-1.4331170979667427,-0.5403086002926527,2.485610444185252,2.9966523255075046,-2.5431679750260647,0 +0.6701141668516459,-0.0788636586975866,0.2864111227706914,0.18626362165384935,-0.7650030967087256,0 +-0.3377963372494488,1.1649050168117467,-0.3872622612113061,-0.11829737091183556,2.051646927287605,1 +0.9980199922471478,0.18559842413835825,-2.2407553155277187,-2.3651008048176574,2.459432866402785,1 +-0.7945504510619537,0.5175828283290962,1.4919501326537121,1.7129151191849186,-0.799062735588009,0 +1.6503342945850363,1.1281449359126885,-0.072040078523959,-0.2750845754297344,0.4419213912638923,1 +-1.2951254681450182,1.721086368160066,0.971925648405922,1.0581178262071773,1.0639110393743985,1 +-4.699895153393166,-3.867973737991636,-1.8321300147720707,2.0036382778827204,4.16786757311883,1 +-1.1635201700965954,0.22842715159760463,0.6933303113526106,1.181074335702699,0.3070140224080816,1 +-1.7088349058434458,2.2649026657377282,2.021274501544377,2.3899689942894033,0.7733826518272507,1 +-0.34544661643434693,1.9234084399442577,2.4622212341544674,2.873721276863577,-0.5193322150078987,1 +-1.3042794657909509,-1.9550651965776997,0.5199582405166336,1.909565958034472,-0.6289610858816155,1 +0.7255006286327386,0.3407928940325726,1.0324980121790721,0.9582394731284781,-1.248419594612733,0 +-0.14759199836789938,-0.5671728353625842,-0.9164920709392783,-0.4243330224333772,1.1457297707141836,1 +0.4664713114853041,-0.5977920411016153,-0.1604283701873498,0.9089816340225345,0.6185064145432544,1 +1.8470474648833464,-0.652897169715829,0.8518165836107418,2.366979107005586,-0.6490386453174093,1 +1.1070375198014373,0.9188974988525599,1.3514320157305821,1.2326130682337857,-1.2366973681968387,0 +0.7844427257243916,1.2343393135935017,0.3180708736184256,0.24555627630196764,0.5046589795595046,1 +-3.230743460828335,2.645890246031403,3.028258154743457,1.4390742877901528,-1.7239237986626987,0 +-3.3310269275432423,-1.0790975803309828,0.7408894738587505,2.2629192947970895,0.8041510592522246,1 +-0.5071823001061533,-0.3990231852180639,-0.522464175290032,-0.6059388750224177,0.318281116896327,1 +0.8926111097712255,-0.4926592574631189,-0.15765732017384235,0.7393700154232805,0.38771652763166065,1 +-1.4898899320281225,-0.9844451125575433,-0.3004114525788173,1.0225108949735973,1.33784180364194,0 +-2.3819202395799417,0.4210664657169625,1.3206598572243011,2.5614231708184123,0.9695511398639456,1 +3.37855832395024,0.8631560091773784,1.0671389074706847,0.9886777063604208,-1.6618189354718693,0 +-0.08540395263657086,0.8020044423531687,1.5844536471039818,1.4700369574009233,-1.234495344396578,0 +1.3804882134633842,2.002848519678997,1.2883336381663275,0.9731013395471988,-0.3808591751124092,0 +0.1862917031344994,-0.12367742278498728,1.5384332037843644,1.6448577560066886,-1.9539694787287054,0 +1.4699335556270825,0.3410673459145508,1.4848480122784058,1.5068824896797874,-1.9459837128276858,0 +-1.3530471723386521,3.6821141057168965,1.0172874178173004,-1.0785584112881903,0.5905749807990572,0 +-3.3980736760193637,-1.389051460988623,-0.20957081593546523,1.5910454119110038,1.9871201895351,1 +-2.39805677797749,2.2279537310758175,0.06492822802247966,-0.1742081358595915,2.6948518061455653,1 +-0.4036034348858124,-0.3565177829855233,1.3140368866206753,1.4557870779149575,-1.6756883060783845,0 +-2.7179989931840245,2.3610511176102724,2.9366865615005495,1.3809659309112254,-2.032413674659284,0 +-0.953012135089474,1.3583173074168262,1.779729866784015,1.1490512672810336,-1.1935638060717118,0 +0.430195940253815,0.7142321002575143,1.5696382220462068,1.5481064023401832,-1.3749961402029962,0 +-1.8926922186910788,-2.255974293880211,-0.8511751449004215,1.0503662710723807,1.5102936352700345,1 +-0.7830025832324586,3.2773777708800758,0.553044030554017,-1.2513888280910637,0.8832949269426627,0 +1.147437207575659,-0.15164732938544145,2.0181749187803604,2.3542375196646956,-2.6392533166627032,0 +-1.3992608764550365,-0.09225318455408793,-0.17773450173630467,-0.08506670142183337,0.6966881883722482,1 +1.7404428775060694,-0.2771524063608892,-0.18802159957105788,-0.34466159595783297,-0.8038006864448825,0 +-1.5158878762156949,1.8964688901442628,1.6487799888862409,0.5728727916781783,-0.7951808583902484,0 +-0.4793989467865878,1.8661102324063834,1.7915363722732964,1.8187619438279796,-0.13337145365765246,1 +0.11986738891266091,-1.1287676859174192,0.964372946980242,1.8512983982440667,-1.3764123411869347,0 +-1.3738662161152317,1.6951663592491766,1.331152261674233,0.2555574969705733,-0.6551459518107414,0 +-1.170378728125851,-0.7318267017985933,0.5396322353868148,2.584179224688855,1.249452883440511,1 +1.281996261879405,1.1903118102593129,0.2842160119974789,-0.04960090040356091,0.04734160685501698,0 +-1.0224528297890734,4.696327212328067,1.7222354592432838,0.7625664477006322,1.8839231436927037,1 +-1.8472825028842903,-0.21762968968901575,1.1271224228707393,2.58009968651018,0.6207710418428616,1 +0.6907254174496904,1.980508959224327,0.7376613699446846,0.6799348242000365,0.7863178631496419,1 +-0.8842128526730392,1.0634640404465578,1.7535956357631417,1.0341805397646984,-1.5779098849594284,0 +-0.7973826752304377,1.7859522340290273,1.823810438965471,1.8521982385260634,-0.1453149260115334,0 +-0.2826395282944725,0.09859698495928815,1.0711407225715162,1.3456531607446796,-0.8162831572186477,0 +-0.3084515798795451,1.0690206942719902,0.9100414468838657,1.0495519036731304,0.21382787328831077,0 +-1.1355309520352503,1.4674397550313394,1.7048894822997167,1.5646992323412887,-0.39112229310443747,0 +0.715393053765699,0.5343964735186322,-0.904369800960031,-1.0109373797242265,1.286277354116597,1 +-0.9861941779691142,0.2686171155894226,2.133891268753648,2.2528525527257797,-1.8831018118826202,0 +-0.7558609272896826,-0.3834623388280274,1.1126521037018748,0.9291113939934403,-1.696045911082663,1 +-0.7426104769036281,0.3158720449654637,0.9244096865280529,0.4601046969261883,-1.0780477985679184,0 +0.19579630074165666,0.0009451448809769669,1.8862833729451294,1.9774667394929257,-2.275162963612118,0 +-1.3866351433028041,-0.1775954601401939,2.361079922530949,2.8332721006203387,-2.0853958415458056,0 +-2.3939927362949995,-4.340499409828883,-2.184367115497969,0.623442260481649,2.2201873685565245,0 +-2.072527784456982,2.067309692939296,0.8302657224862183,0.8935016750551722,1.820009079736349,1 +0.6600780684137897,-2.125385412574427,0.5988008440250892,2.2359918077627396,-1.2825659644655572,0 +-1.825703045131993,0.11745027426142596,-0.6943244047738484,-0.19797047267987855,2.127364051411047,1 +-1.437833344864777,-1.509161496878043,0.45190534603338706,3.341503984088233,1.598038190654035,1 +2.6245648117759495,1.738039793433062,-0.27670534494328836,-0.8404467084192622,0.5800025045465871,0 +-1.1029519523743057,1.2736497747866429,1.168507592794759,1.516147152872163,0.5980202645257587,1 +1.0851713179196727,-0.8940644829880732,0.25409267691875337,1.5353133959537153,-0.1605866611723814,1 +1.9273121596147358,2.1300439481565157,1.6156534052497245,1.2597263026138057,-0.8825940008517918,0 +0.5479971926153351,1.611530174414562,-1.776851225392906,-2.6645291202832464,2.6308279745600247,1 +0.639926019310195,-1.9996095478109273,1.1278844023002699,2.7625881544574193,-1.8006352295592567,0 +2.1865827375994966,-0.006849517706070785,0.3419925712784925,0.3621522679659537,-1.1371120561820225,0 +-1.2352605144849562,0.2745363641848424,3.084890003129212,3.476508657470033,-2.658407502092991,0 +0.6571720851343019,1.468817091210764,2.0427483319540443,1.9707915331831325,-1.3298577000289156,0 +-0.4316822231372386,-2.1957079299533353,-1.6349398459031432,-1.268592272553207,0.3501726644327826,1 +2.210268547371121,2.0023117800906043,2.4050951668259164,2.3178060151384043,-1.7768440285459701,0 +0.8922197579307974,-2.7758127626357707,-0.13094048489670956,0.3573819070766706,-2.3836215699559444,0 +1.1064405159504629,-0.3840921824342858,-0.14095146974397554,-0.23359848205849776,-0.6855181255769481,0 +-0.14518075720736567,-0.8175471658002196,1.5609818803478932,2.133649146440424,-2.0524994453832996,0 +-3.0644249018002396,-1.1087104854524163,-0.5509751235502053,0.592151467803766,1.850409117189069,1 +-0.03804872636518175,0.236868710669764,-0.27106103611700094,-0.8064002285970302,-0.007263030613972266,1 +-0.18427236611124376,-0.7727342178518397,2.084737836554071,2.2322528984432424,-3.104302623032173,0 +-0.45304295415602946,2.885506207814581,0.7397198829925555,0.5618894458297679,1.942383041963169,1 +-1.3020500920184763,-0.8121370027867627,2.272521488797995,2.33642578704117,-3.090972539545532,0 +0.40604455225384317,1.866917035870034,-0.3228644415023857,-0.5347473807078809,1.8974116954692772,0 +0.14950397418466688,4.315398480194434,0.9374393206917495,-0.09687077769368413,1.9881884393786664,1 +-0.18142672451533515,-0.7450424847562123,1.525748430562944,1.7531614396400332,-2.304705637817097,0 +-1.6103531561158335,0.34517283649544117,0.6395365554618206,1.2302743007532913,0.7532515540173866,1 +-2.6522815459157147,-2.6192061099048214,-0.9065792709661638,1.6444366533021086,2.1840889183822902,1 +1.3720256775654263,1.1766746161370232,0.2793159818330495,-0.014060019360358611,0.05400969285935742,0 +-0.5221182946265343,-2.4327349579154673,-0.5113692988648673,1.2960015295108884,0.3533609376994866,1 +-2.0152667861246716,0.4057320307490234,0.9208210776206208,0.706085293876693,-0.2818326993341542,0 +-0.6354085171375272,1.4872379682098882,0.11290070766951468,-0.5519271292728085,0.8335199558102054,1 +0.08151836346102226,0.5971028523175346,1.8921982281223508,1.8678793416914101,-1.7736762191643702,0 +2.1890654811427477,-0.8559419336067196,1.5373306010530063,1.9354903115384956,-3.037198623555004,0 +-1.1421652143495917,0.7126294295398583,1.958633687268007,2.2468010687592415,-0.9846585409573603,0 +0.42056496129497434,0.6077443489230427,1.279508535138206,1.0710704445248804,-1.3289068326792008,0 +-0.2741334290683849,-0.29617504495605784,1.2265628479410087,1.621924733037381,-1.272180161427611,0 +-3.1952380625774794,-0.3384360944301108,-0.18241922331834926,0.9953245805587717,2.2530325624622654,1 +-3.3374933326669134,1.729835890338284,2.827853264146113,3.9972984492986425,0.6770481400283077,1 +0.6563819780401157,3.213813638896659,3.473594037748276,3.4079701408041196,-1.326315092594193,1 +-1.3554056309101163,-0.3508671887713406,2.68805675893081,3.2690385190737494,-2.5503300020622603,0 +-0.21023058406542605,-0.1974380049698623,1.0825160849850666,0.9521892749728701,-1.5970819883753347,0 +-1.2516282309921252,2.4529504728183804,0.7027205930687741,0.3716892207766668,1.6532823124091673,0 +-0.7850626346700532,0.5945250975698122,1.670024673002114,2.4740763184906815,-0.30115445641480587,1 +-0.9999878637054567,0.09526884694013305,-0.2500755626644043,-0.12004413558434068,0.8804978405757139,1 +0.31512629112183466,-0.6383682723118975,0.8483766728534774,2.2498799033832344,-0.24155110927032108,1 +-2.8653361653609215,-0.32433917505990517,0.45031211737780585,0.3222668774170927,-0.056634118717002635,0 +-0.5947029314404539,-0.7672570851390981,-0.3944608502790167,0.4350227528354056,0.8270903875758379,1 +-0.13609895481961676,2.6577948524744572,1.7664622562818162,1.528441658108402,0.28368104771272185,1 +-3.1635478303235485,-2.44051158074097,-1.9624254902183873,-0.15429922452392608,3.0101109958355274,1 +-2.931589374516034,0.7630246425679839,2.8110573188590706,1.6585207502575812,-2.964735774782981,0 +-1.7731933680370298,0.7382986132046683,2.12999687874008,1.9020113496553541,-1.5255939898621205,0 +0.848605734550433,0.053340709386014096,0.9129278568227747,1.3904254099771445,-0.8237606750362854,1 +-2.077749694981592,1.2060838415611133,2.529500051720093,1.7599159195584915,-2.0403724890298385,0 +-1.1188689268961265,-0.20285904972194568,0.794779276879819,2.7243104562149902,1.3231814713383816,1 +-0.0954175913191071,0.26825425209160625,0.7833150023168168,1.155335397728294,-0.2490306969852848,1 +2.5390054121343146,1.6099634285770086,0.8634401458207299,0.5942047829519292,-0.5921285368225542,0 +-1.4794498803961207,-1.9944796471316764,2.6921534412649595,3.4204451955460318,-3.9990891618057414,0 +1.2774871040329563,0.3184220525939194,-0.6303355119697646,-1.0548169993328025,0.1954172762731774,0 +-1.5311024724204771,3.015082305734303,0.0320953116205837,-0.461751926828889,2.952786971687642,1 +0.7087423407483889,2.117191185543434,0.5234616899314095,0.007205425183772318,0.6747459452722488,0 +0.00047988707795476593,-0.9687107196567223,-0.23968485234385523,0.7714105907167034,0.43588539521221004,0 +-0.0161488106623352,4.669858118845295,3.1210033665177224,2.17567235260933,-0.17798132744404005,1 +-0.4733574009373662,-1.5865198933906508,0.7566849118259822,1.052750114602189,-2.032518381890227,1 +2.915132744938017,1.5785145276602148,1.1804433268067656,0.9325964531999318,-1.1146234531812436,0 +0.10093815568700482,0.8406918011448388,2.1656653730016506,2.1528204483240563,-1.8583976901061368,0 +0.26660966057789126,-0.5327505054580786,1.6533728603776932,1.9365706768947253,-2.337111160745244,0 +-1.306766266794749,1.3721033307752952,1.9451614090689782,1.7865535695818553,-0.7438827417376146,0 +0.2129870676981067,-1.7808155165097936,-0.204253218019261,-0.15503541668157306,-1.5521264634547975,0 +-1.8387596077625608,3.4837695802186133,1.7645980307905438,-0.353269844830417,-0.38508385382087695,0 +-0.38110704059262157,0.5458493726301588,1.1278493163692698,0.7499770883061763,-1.1228271281145976,0 +-1.8580951731542594,-2.2307726004159716,-1.5009570089353417,0.00166573101847578,1.8806794525585093,1 +-0.4437113239703483,2.4614639808671477,1.7825145718053808,1.9602162609219822,0.6282088714915346,1 +-0.7844587082983987,2.083342056980249,0.9435954528530927,0.06979558617981219,0.23326387485465294,0 +0.38226677817715304,-0.006200059600729668,2.690762213998934,2.9751146425417647,-3.117700630118287,0 +-0.5309278522168592,5.066865843663481,-0.4661101062627375,-3.1374154970101413,2.8864483947947432,0 +-1.410019303549808,2.4696401250342053,1.3716439193152965,0.31234784762704115,0.10166525901384227,0 +-2.968130722244831,-1.0600420652531284,-0.15237497138197664,1.0800766359299163,1.4769450395539705,1 +-0.8182210711725721,2.5262596157630908,0.5077096281967457,0.16726334750541338,1.8099195616385941,1 +-0.3433153061155022,-0.2553739382456953,0.3278599093016218,0.989505894495173,0.18615097927577462,1 +1.064763447816254,0.6388726488478786,-0.6353496895648167,-0.36060989272132415,1.364150901970055,0 +-0.8627510786187689,2.73870902356454,1.8694695233951253,1.3650012316407736,0.1889272358993046,0 +-1.5144949861995352,0.9464555442313377,1.6552408779580707,0.5418574089732152,-1.797136808573561,0 +0.7872459278367461,-2.3465109064283376,-0.17970963188890554,2.3866044211085935,0.4300424548141356,1 +-1.368486486296126,-0.8298239807292969,-1.5983665977551478,-0.8203610142958142,2.442124734289842,1 +-2.1900802833863624,-2.705222886595946,-1.4998309254692441,0.3575253271372084,1.9057157234870756,1 +-0.5944304413141811,0.07648106833299984,-0.5128673803000896,-0.01505388520269535,1.4525796937476831,1 +2.205311501013434,2.87764561316981,1.1924468989377601,0.6428837203925529,0.07877766318581303,0 +-1.1694690187474925,-1.6922218515468277,-0.5413061895205908,1.7843306832511756,1.920220232124858,1 +0.02169045265202829,0.28577081533522497,0.57274304642899,0.5174921281254792,-0.48331638566901236,0 +-2.1262731706614,-1.4908729720098677,-0.20213130075300634,1.6528753743755034,1.5092167931068188,1 +-0.5166990351338983,1.1125629463304683,0.8392744352190076,0.3207341333975612,-0.3106461994135301,0 +-1.9018373806185203,1.0130998539124594,2.740807673679642,1.9705143780720435,-2.552480088833656,0 +0.2746188684647839,1.352856566280071,0.1626158380649908,-0.15765320572363017,0.7120873904467662,1 +-4.004869310053971,3.1486777336534466,1.765675716827526,1.8344939549015717,2.412374619697631,0 +-1.1602302676592828,-0.5614874171348887,1.8944876428836814,2.2391469135236113,-2.1149915158449666,0 +0.06459193737589453,1.3638540025793289,2.212482559165283,2.4090888558127466,-1.148480917733877,1 +-1.593087451627724,2.7744621112437753,0.3555648732548897,-0.22875078352965916,2.2364760973958617,1 +-3.128584378019733,-1.0911466013197306,-0.4956850484555142,1.1516239114172049,2.3770286058830763,1 +-1.6982642305882938,0.013738420719339839,0.2870613090244032,0.8021998227489997,0.7990647255364014,1 +0.24768995621618872,-3.6934560531095864,1.162625432150056,3.7674016087104385,-2.3412683756059005,1 +1.8122794244527891,1.2840765729053571,1.5737684484570966,1.4381080921515934,-1.3981342630955338,0 +-2.056981582377505,1.000746241417191,1.159502794899801,1.8753955225019447,1.061050481965919,1 +-1.2661610848654723,1.4731728054260682,1.6357169743202529,1.1128825402142797,-0.6781976103932238,0 +-1.3668951333908024,1.2916832621812082,1.4534160095360353,0.20404110076973103,-1.4030964798775367,0 +2.266331755156444,1.4100671777467448,-0.9897002915160751,-1.6749763110319038,1.11104323108902,0 +0.08521829392459668,0.48582146865581166,1.0359831897554665,1.7983239178119481,0.02876550234356756,0 +-0.23818963674681393,1.2416373485393306,1.2963885061556009,1.627663364110775,0.10116457698095749,1 +0.8892306496313445,3.397805282183613,2.0878568468383842,1.3990176426074203,-0.20866019982996742,0 +-2.3794062934004385,-3.0696846568674827,0.3665201039187584,0.6219533385284765,-2.4466459256613127,0 +-1.181088819932628,0.6550058679807468,2.6798044816311126,3.7145470996426213,-1.0906966858826856,1 +0.10180056002122373,2.4380012669868725,1.7698332048950611,1.4539894463346454,-0.10627626682031432,1 +-0.830728129885048,1.5652155463439628,-0.04665880114874221,0.10561167914707081,2.0724992100898847,1 +0.4053469140716852,0.5085149521854901,0.12557040548966208,0.1664542083900059,0.2649581900443695,1 +-2.342044763600879,-1.6102245339139136,-0.9009841600558268,0.7439656217456587,2.0867839688512593,1 +-1.943594805398887,0.6026664336902904,2.149408914681342,1.8587993688549305,-1.6971381180693061,0 +-0.22145061426608326,-1.3802511151583472,0.8970257164432489,1.2003864539383198,-2.0741477948296745,0 +-1.1107140746496205,1.398575537620811,1.594169886120472,1.375703947841755,-0.41903729343637486,1 +-0.7501284788110372,-0.9460285041440755,-0.006853932826404896,1.1213081086011598,0.5540924679653598,1 +-2.222054396135138,2.592358408534092,2.39211361201816,0.8777125969910466,-1.2542633552423696,0 +-1.616243234686673,0.9938414758984357,0.7773390100652748,1.1467664435297087,0.9929247654478617,1 +-1.7244738756261175,0.5295888741918227,1.1348698041257732,1.6026990693206526,0.23421918111137052,1 +-1.3397535333995925,1.4524301538941007,1.5675697451738215,0.5166817054168771,-1.1723322419562998,0 +-0.4856852714574713,0.18691306601596092,-1.4252170959616148,-0.7689693778204301,2.8189745204539434,1 +-1.0478188489343447,0.24228945360911802,1.5879043108308486,1.4141583763716645,-1.5422850563431323,0 +-0.3026220797896213,-1.733485642705193,0.6392241515257823,0.9743323085100991,-2.0502163270292275,0 +-2.183849983904972,-0.3101082258850044,0.2771304783973557,1.7368409938398748,1.689691815913494,1 +-1.020139206182745,0.13171892378542494,1.4722186987595505,1.4041244027433903,-1.4043352332824608,0 +0.6013660904301237,2.8004285616392037,1.4205178137451582,0.7848773410218395,0.16521364314630993,1 +0.7634360370600045,2.0248846181685733,2.1311332107291685,1.8295655782622036,-1.1692307718720498,0 +0.47307641905248743,2.353525104688271,0.5460729612318432,0.027616086347507784,0.9605785312823023,0 +1.1053304543028952,1.8893127422602787,2.162096339685987,1.8783357865679275,-1.438138685322984,0 +-0.7138213077743706,1.7869040529123357,1.4205822780176687,1.5941940893394366,0.48152184816815435,1 +-0.8891323424498941,-0.36308613733585393,1.1333748994243873,2.778566473852072,0.3575714690392612,1 +-0.5547510768756081,0.1081850992932003,-0.6733035177255997,0.01589911541387834,1.87836229164553,1 +-0.34186297013162203,0.2297043108749648,0.10820640206396333,-0.13168373794546073,-0.05180382231499925,1 +-1.1907899668219804,0.34244813705098043,-0.14796631363379606,-0.1862098889597057,0.8818734473676779,1 +-1.876489014446142,-0.02316286704502568,2.690727492332873,2.6942024583583617,-2.6862865240547755,0 +1.7285088360239254,-1.6487589150274347,0.2842422345853489,2.2321338847827787,-0.4356753214496414,1 +0.6651067710198211,0.30819928604673197,-0.2512765521375714,-0.898994363670609,-0.3196241166145162,1 +1.9782040375531553,2.628504187663925,-1.6040206174001896,-2.6537904721434713,2.7802638230733763,0 +-2.6510828902659993,2.326386801219159,2.7204071772018747,1.115068251448843,-1.879290559542157,0 +-1.186834146569784,0.400025501518365,1.7401439218567984,1.2164498335057194,-1.909419390148308,0 +-1.721182046850616,1.8328720498254603,1.1220953073737008,1.1403298533081758,1.06007742179251,1 +-1.0976449177973853,-0.5588748375335651,-0.11428676794601245,1.046114122273532,1.2258684905247472,1 +-2.055258399352235,0.7297352574254683,0.25219838395878286,0.5767320764466431,1.469347481897939,1 +0.9396325057306661,1.8240610018996333,0.33292806973467104,-0.12167131419750987,0.6047759997645985,0 +-0.09091993774665363,1.2264744367918556,1.5612489629809438,2.1151270225387098,-0.04277116280580451,1 +-2.0128888578762956,0.9541551446895393,0.2412095201108238,-0.018599588426386093,1.0500559717062075,1 +2.381806116417988,0.4540864286745504,1.4210397807376298,1.4490533405462178,-2.0538209842831057,0 +2.160066151875852,-3.0363564548476267,-0.9412889899282006,-0.5780084656984401,-2.2150464752320014,0 +-2.29951734855831,0.7159332151564084,0.1063868128596227,0.37019985974029024,1.6492103469246384,1 +-2.1169951863111063,-1.784911689560245,-1.1179894043374845,0.557798912758207,2.1359931651711097,1 +-0.9340102532402836,0.6630945859933702,1.6881548302857148,1.4345176531864965,-1.3694364557519196,0 +-1.4213411477185902,1.7459448795822081,1.3604699327788818,-0.2257296245975493,-1.186534886247403,0 +-1.7296842285076708,1.4741464200540437,0.20821073658789635,0.48095677044851903,2.1031983283215974,1 +2.5425864183505036,4.092218706225297,2.232066623101205,1.6319321340706483,-0.14616646226927688,0 +-1.6608127328114428,-0.978266648347326,-0.7344388104076814,0.29683572506910316,1.6119138738266148,1 +-0.9874175244628917,1.0977031383960472,1.3314180616152052,0.05653674473840642,-1.603469679402069,0 +-1.3501854845298977,2.3902491287641086,0.19801646149880547,-0.22970689511433728,2.1353117390007395,1 +-1.2824786246923263,0.8626410830246449,1.6330904953313787,1.2824759378933648,-1.091854168122208,0 +1.2087260529574664,0.9001894851085859,1.9538468070830133,3.065143434466759,-0.6729141903713469,1 +-0.623248462831707,-0.08381826763728495,1.0679325456724942,0.9470072258302213,-1.316424216195174,0 +1.7931884976559327,0.12649432032333552,1.5489265183842555,2.974289255357829,-0.8026077354330869,1 +-0.16763373601916265,-3.1031377831109586,-2.0402743754390076,-0.40379116317383734,1.2472732826269155,1 +-2.2159511427157663,-0.02821050295166927,-0.7730512531149942,-0.13380289603564655,2.366066770059766,1 +-2.4999385413196746,-1.0101783866621894,-0.5480421839854156,0.7704605091578789,1.9493626188357425,1 +0.03854326291922672,0.9577933941193836,2.259175626413194,2.1754709544095685,-1.9126748390242563,0 +1.3362754068776401,1.2071015263717277,-1.5656871389724607,-1.8426637312791994,2.3748365772541424,0 +-0.9878438890073911,-2.2283478993005166,0.09647884946811158,0.3778953737880706,-1.7106384833841974,0 +-0.6995921000121499,-0.2781565433820208,2.2205122182070376,2.3478967053424897,-2.624174237941769,0 +0.38942275712397234,1.1086167968945657,0.005346014015365552,-0.038132509973745954,0.9262293654372233,1 +-0.5483603760983984,0.2246045803547324,1.5757524876203959,2.4386904939590925,-0.5710283439117549,1 +0.2887984783310744,-2.225765327691778,-0.41124288271135157,1.4037684970935498,0.17456258033499705,1 +-1.7194675932886905,3.49563582725577,1.4910663788056069,0.0071082955017790495,0.6199800268409534,0 +-1.641095771570324,0.36700782206395566,1.682218549686627,1.655567895378898,-1.1718092871581167,0 +-1.0837360084944896,-0.012912508425786173,1.9818895366840326,1.9551630985619182,-2.106813149627893,0 +0.5682995631211969,0.6795301163408718,-0.43693465328778436,-0.20465566737967356,1.2816091483084093,1 +1.114623177999174,-0.4797482353480105,-0.19865453171293468,-0.2685196734349298,-0.6883704507870884,0 +-1.0795540158597503,-0.16818533476983055,1.7418841571821706,1.659512082223742,-2.03111672915667,0 +0.6607240777221002,0.6029286424214246,-1.3466800499329494,-1.2601089442740192,2.127870302820532,1 +1.0344020171889092,2.061167561981364,3.2066453961513783,3.0462033259709305,-2.3859289045451915,0 +0.37136626766904346,0.8988589559057312,-0.8181199281274159,-0.610500441496342,2.0073530474906947,1 +3.7116398922112634,3.5889849091355615,0.25090872873830006,-0.594608740422508,1.114082738598523,0 +-0.6197162938109735,0.17086632906781907,1.3916354379240317,0.6515852235281958,-2.140677563898946,0 +-0.3151282280596497,2.3140330376379397,-0.09883021121604015,-0.5162790200935368,2.086682301716092,1 +2.709356609461822,3.227611268129213,1.0996805111753503,0.4119111496268357,0.22195526048371694,0 +0.33384529347991454,0.7616572175642997,-0.09741462686862226,-0.12735033977625054,0.7378804810745756,1 +-0.8423817325854446,-1.0294860412093838,1.6177786440375432,1.9756600339212629,-2.337187834480374,0 +-1.2923255145001433,-0.44032485980351876,1.9596213213023441,1.6639831976262847,-2.7341414527610945,0 +-0.8246931696906928,-0.7298850311993501,0.05459299495176706,0.9840113582527745,0.5016273852505465,1 +-2.248678031706808,0.9247170757928248,3.1083972032029283,2.806035802350913,-2.459750586635631,0 +-3.8332084286955097,-3.1927848638074052,-1.9961075204809295,0.659131258664146,3.454796585664818,1 +-1.2838085755994462,1.5916029568860783,1.801206241737078,1.1038237210006263,-0.9485190719402472,0 +0.1955527031374109,-0.06374443514060057,0.43929618385084285,1.2908962891709552,0.27014857744842047,1 +-0.5150116318388791,-0.7317942871667593,0.20224014659876643,0.3276870790115657,-0.6704129204329644,0 +-0.6183484211323023,0.5910614737199885,1.0104138016738153,1.121662483059466,-0.31541706411392356,1 +-1.1117206206580046,-0.2916144896590832,-0.1403428277478413,0.5031747723035098,0.9612026352836874,1 +0.6806985930859077,-0.37924856391598605,1.292181110236423,3.306184317593251,0.026377881143267112,1 +-0.7664345529320518,0.12088079144993336,0.35907121412088133,1.2259676015461236,0.8930396756946153,1 +-0.3622626006601888,0.035774601592219235,-1.3191452563411281,-1.9347799572605067,1.0954755205955942,0 +0.7438074113115787,0.11163159415571533,2.173713671915017,2.3461213037236375,-2.610714833688412,0 +-0.48135405773256246,1.6487039054926758,0.45031714530886136,-0.01355253185336891,0.7516396829527527,0 +-1.3313860159229876,0.3298098086687592,1.6820742692246928,1.41915847628876,-1.5729955837822076,0 +-3.201839618683057,0.4333381218907446,1.8075318760883068,2.9056901374156774,0.5033912670822451,1 +-1.663699797264159,-0.4778971833613952,-0.8685272153745132,0.07934271843335561,2.186878245479578,1 +-0.04011990176560687,0.3467979463488742,1.7792865631682049,1.8640410783385772,-1.7253397232117011,0 +-1.9700983212922178,1.2523552920846863,1.949547833863725,0.23508371744911494,-2.360153340043535,0 +-1.9138076073277235,0.764395733894055,2.8980653256459368,2.4929620747337333,-2.588259348106957,0 +1.4499153903806032,0.44282039190991374,0.613886141302318,0.31144683650722127,-1.1275968948981936,0 +0.6470644837194861,0.30046407669196873,0.12847765759192872,-0.16581784002844846,-0.39735177152483947,1 +1.6947265626501289,1.7341042770129143,0.5895345458275246,0.09213992065230914,-0.10018132696248938,0 +0.6742874243923125,1.324588920694254,2.251732728997325,2.086523998891481,-1.8388899745731457,0 +1.1406781298884257,0.7317402925321221,0.9674071953350729,1.1902859708937874,-0.5888381734691674,1 +-2.351107596226875,0.3320393505891239,2.5383540909712203,1.879936816264486,-2.7132526067228433,0 +-2.2922099781416563,-0.7762820456223478,0.17260607641965087,2.666820189194512,2.526030997535515,1 +0.9883565057274546,2.0301834171146,2.5645732584655567,2.142848439045399,-1.902687974180868,0 +0.2506410561359709,0.4043234434677987,0.25703804342676134,0.4259526996659099,0.19234538233600929,1 +-2.063942516778731,-3.2362740435065613,-1.0650128942976698,2.238045477366481,2.390451336254662,1 +-2.294813314784948,0.036749828525122874,-0.4527629649810289,0.23352676724312027,2.1170630082453634,1 +-1.2302965864704927,-0.2298466480606438,0.24753480645585169,1.3628471770861816,1.1072737967808348,1 +3.0384155702296543,4.143770640290862,2.467960273050572,1.9479267104159412,-0.46159179072637213,0 +-3.7094732863027984,-3.517098886014722,-1.9352302476009529,1.1939679596310857,3.535558279763149,1 +-0.4999238365897466,-2.1254535517533215,-0.21641095772404007,2.0467239147560097,0.7944964700137268,1 +1.0971001707655608,0.01310228471775654,1.554447681608008,1.6614300046149642,-2.1413912055401263,0 +0.7521860091967676,-2.160039793400006,0.21983101732403232,1.5726801364453347,-1.1970617727144066,0 +1.627547479262255,0.5168697864669995,0.6436096857032132,0.5437260884800762,-0.9263060161592033,0 +0.8451155041244856,0.8960552860738265,1.1229768859120794,1.5928153298949423,-0.2436052520647194,1 +-2.0832120040552278,0.28545220681213024,2.6319185654520636,2.468367033417705,-2.4194740744165673,0 +2.3189355661288777,3.657096913572673,2.532287999620387,2.120658582694909,-0.6675199660742203,0 +2.514804439945528,0.9589232982463185,-0.007801500132298411,-0.34885395731874036,-0.2483678033789718,0 +0.04336559759654013,0.19306473938811186,0.8100341524878176,1.0715254475141178,-0.5254077854088179,0 +1.601712940618461,2.7790084526340104,2.244141054750555,1.9260173812180148,-0.8511609401948771,0 +-1.7845742746585624,2.532392934751994,1.5644199644143153,-0.1512646945552234,-0.6687955168998937,0 +-2.167829291127316,1.092778028707174,0.5781339787714421,1.3682497433502991,1.9844476037061105,1 +-1.6069174440626608,-1.097467148749626,-0.9772245360785796,0.4436156961591704,2.200784328944952,1 +-2.9496481104082717,-1.6069617663602425,-0.3521390305195575,2.0663626166008195,2.473360138385329,1 +-0.5409518022560234,0.3574619516351627,-0.9704789289747868,-0.6606489286947119,2.0698939959991405,1 +0.13160598942733448,0.44437593117930385,1.0959011994986412,1.5256800383009055,-0.4679753514024687,0 +1.0350492547187982,1.9223597492533595,1.294083983295678,0.8979778772937145,-0.4417722309115306,0 +-2.0997817197359843,-1.3401159845072177,-1.878468194513212,-1.820871937773612,1.7257149891786197,1 +0.06841677105976451,0.3447698517857547,0.5021144707531356,0.08012531196973083,-0.7571906372960562,0 +0.6919403446265544,-0.34293463887676756,2.0482636912577834,2.2489084878756227,-2.864196127444556,0 +-1.6565996208480873,-3.8752860887893075,-0.9494448204428232,2.5826560357572994,1.723908612238768,1 +2.563877439284834,0.7863044584852843,0.7058859641405053,0.4356129097848864,-1.2342424504416118,0 +2.2348679265837403,1.3680263865364317,0.6826059929516554,-1.0114090614468005,-2.0801160844972273,1 +-0.7389636816417426,0.3463081423092159,1.051862657745989,0.6638354525128904,-1.1209014274821953,0 +2.324447412224612,2.124082245266079,2.0822674998350155,1.897853448248556,-1.4045492573508662,0 +-1.6577953997060195,3.5431347050524415,0.887952542659506,-1.330970560162208,0.5763947111639605,0 +1.591527790716269,1.6389674367702853,-0.28904557679363535,-0.6074263913873148,1.1125374029502466,1 +0.9768572261504459,1.856721989004823,1.4730920850173237,1.1880251287815673,-0.5850662427303743,0 +-1.8374019353513755,-0.8124716733825097,2.405578214946731,2.451563919134256,-3.094464479303658,0 +-1.7266218841437064,-0.02359013524517739,0.21001239804658728,0.700480313093474,0.8383813812055708,1 +2.1802838750955535,3.470134620610875,0.9561459732068427,0.18220439725336002,0.7234829450256977,0 +-0.61362910569733,1.277457380445354,2.6041062054346176,2.4705938784249146,-1.8508896607627023,0 +-1.1551992864958427,-1.403366410995885,-2.277493613161628,-1.5239670534588141,2.6006806747495803,1 +-0.24768481415092636,-1.0577788044260061,1.6549735985515848,2.5071806681958346,-2.066232394376576,0 +-0.40792379960268454,-2.11614855382016,-1.1867980283321495,0.6400109339000085,1.4812640817047842,1 +-2.6807979253862637,3.814353014575204,2.1489538718525942,-0.1856619125503356,-0.48080369359806185,0 +-2.1564998164785916,1.678196980167825,1.301903311756722,1.583754600957698,1.1210811869866673,1 +-1.0667807573843415,-0.2295574959439297,1.5376945925285677,1.6738964511428893,-1.6060707437602013,0 +0.707515555042219,0.09998639968317125,0.2507098656716912,0.09808059306062167,-0.6123262458677282,0 +0.8938415693004338,2.5214848074697316,1.2823021129819532,-0.12641336366068656,-0.894528237932827,1 +-0.7336436558532282,0.5185386544516029,1.5954444146761273,2.281756873404877,-0.43286474248998164,1 +1.9933451428865494,0.44786633653555363,-0.07911833108975108,-0.29277239663272425,-0.3580705516023711,0 +0.4900666484823464,2.4673661249058867,0.9954950996161172,0.7506916833444428,0.8197752522728632,1 +-0.9467154611854431,1.5903687354854186,1.197831867026652,0.5645744841105107,-0.2530340452560509,0 +-1.9596413035128326,-0.1460075894234103,0.15328380099335082,0.5392273077609883,0.7482246255974008,1 +0.4314459009784185,0.23459775906487124,0.8853239912964086,0.8412438631811001,-1.042697945149408,0 +-0.682381858468813,1.5830405886634114,-0.4626840275989676,-0.3825364806020919,2.470844271870866,1 +-1.8871383808349407,2.310377977420807,1.7028298713560945,-0.23992216781650044,-1.2765598213019305,0 +-0.5286893495491088,0.8307116363396054,0.4096662562418744,0.8083305098283353,0.947211910657629,1 +-1.1086987474820493,1.3403732787172116,1.2708450037571506,0.6790361664064749,-0.4930976132004564,0 +3.7997862565231837,2.8231018862268975,-0.11205294943640298,-0.8358515267222777,0.8954838892407861,0 +1.2439449268414506,1.5504099503577282,0.6516751482305945,0.7993903769664397,0.5011880088559257,1 +-1.3208756567823317,0.5564960244888131,1.813863712236575,1.6260054751303046,-1.4280846947878518,0 +0.5665198988975316,0.7076262142566467,0.7347880628058787,0.5450223673830455,-0.5898556651015592,0 +0.6447791789033059,-1.4895538960441748,1.043697819419559,2.509025163554478,-1.374367048452404,0 +-0.4521339707611136,0.5376527171020264,2.3082890830940848,2.4634897292946816,-1.9663085486423513,0 +0.2890182638735428,0.35289945900642006,0.442788495059652,0.2980419011250631,-0.44505143220716015,1 +0.6464509670047949,2.1225867116871306,-0.6063456200634835,-1.0136030556116213,2.2051870097694124,1 +-1.0043642193681748,-0.36903880276259327,0.4772291711960822,1.8855376757923101,0.9332236243286268,1 +1.4784095104922712,-0.09418865070027349,1.289492743189909,2.75370098018944,-0.5576264030407259,1 +1.549390742441036,1.7199451325613535,-1.9725175483450563,-2.7910439285924404,2.7194507122029585,0 +1.6928486117645272,1.6227137580803586,-0.5475492099075668,-1.0739314699657307,1.1498847530316998,0 +-1.8481648613456465,-1.3680414216592993,-0.2858285550585581,0.8807837107948446,0.8835023260623835,1 +0.823472612805096,-0.4060354773498627,-0.27707799752811246,-0.24382068734272155,-0.30717739220800033,0 +-1.0585819992207868,0.3382306232119072,1.2905920518937077,1.1998223693680459,-0.9868943631319809,0 +-1.7864343328060586,3.043215267226168,1.5400007821494748,-0.33450392032894416,-0.3011015859669931,0 +-1.9895029416000818,2.8699187499026335,4.156990687365398,4.629231263699175,-1.0283862390780407,1 +-2.53345323279635,2.213763631483755,1.0230933022578965,1.1828624489486188,1.991508495570377,1 +3.694382734619962,1.4258867679159382,2.8551143719515646,2.9135921037866885,-3.2432821950263593,1 +-0.3229596889549351,0.41994802499869033,1.315193731827938,1.4319699750443637,-0.9533220376228855,0 +-0.6931680314488304,1.486994519060338,1.331380609980207,0.8156047764102605,-0.4759410493155619,0 +0.9246407877180507,0.013112477663526856,1.0762159780321128,0.9504509778663759,-1.753969419222451,0 +-1.447861253324464,2.536240631419267,0.0924883981082443,-0.6043225320200212,2.147370120503993,1 +-2.3747239989830833,-1.5618373652740922,-1.3289876923270154,-0.18890172598861676,2.1146115977457782,1 +1.509564649318543,-0.22481104481758962,-0.05458564756812251,-0.5609032482918916,-1.2224348551211697,1 +0.33793606237986684,-1.247233208724829,0.798474969612832,0.9685825688647349,-2.1544641452547646,0 +1.924751621477594,0.5780229235918963,0.5940248823663592,0.3181062295247187,-1.0978128016688715,0 +-0.4460891711101792,-1.2951712461677676,-0.5649038941383058,0.22338593428913478,0.4114939359639308,1 +-0.966500140440298,-0.6366280603084185,0.9163111487378174,2.2874505456409953,0.073445795142561,1 +-0.17400538786312814,-2.903320096389185,-0.1333606829552334,2.6655246561308132,0.3936740937102483,1 +0.5111255997619428,2.0182005425810368,0.23740435037226348,0.2128302203626108,1.5337093780153723,0 +-0.26053173704652377,0.5421592810550548,0.7667217505917358,1.1409707911227833,0.10370361678606721,0 +-1.4347634685445718,0.07826675875326439,-0.13708913697587322,-0.3133481960380713,0.533581416407303,1 +-1.5034976734814962,1.2600133231048716,0.6624841303248934,1.0525433707842982,1.385345110412152,1 +0.8038440401342939,-0.626507843215955,0.45114038743550416,1.860911483240411,0.1021506282373521,1 +-0.9515292283701242,0.1881084913634249,-0.014134605829422942,0.6716553705745629,1.2802834845949889,1 +-1.4572742036743271,-0.4183726926174801,0.8219694102053441,2.833545002142179,1.2777053580363342,1 +-0.3926526997626799,1.2007654631451177,1.2153381229573146,0.8759558687199365,-0.5272535031989258,0 +-1.9382844246305173,-2.461563292115569,-1.2740289215648586,0.8721509367474011,2.1069378217632826,1 +-0.21989713609115835,1.3415421889102033,1.4450916681688357,1.5272120985159436,-0.2613779577393751,1 +-1.4141777246533,0.12051094176474453,0.5838403206840815,1.206929589487416,0.5661011228254149,1 +-0.016880670289741828,2.2767030498369234,1.6714025097050955,1.8540292092945618,0.44136896979271867,1 +2.526449395167875,1.068622491894829,1.0035795059678358,0.8585360503267078,-1.165479098302503,0 +-1.433969747690902,0.8212763224947677,2.0251327907864454,1.7015245575969753,-1.5330440079588534,0 +-1.1282998159566955,-1.0725098741935613,-1.3547273924529657,-1.4404886545780546,0.8685921722859722,1 +0.7401986072212519,1.6365521707887303,1.0812963739732364,0.44075984890115316,-0.6378574733028322,1 +-1.387173275186461,-0.39614606721823775,-1.2269272525396557,-0.8043747360904905,2.036634525894616,1 +-0.795996076389942,1.5250704357767093,1.5384306331851958,1.0610917478404776,-0.6146045049495701,0 +0.22263625812384524,-1.3047802076182804,0.9008960533580123,1.4881603410856379,-1.839557211082197,0 +1.5443858592299349,-0.6255963628661819,1.331388255945231,3.277960524066786,-0.632573013481428,1 +3.005767501435023,2.5187206196764462,1.6681122195231597,1.2509193445115503,-0.9864777212480507,0 +-0.36352340972671116,-0.16684293194668987,0.167955596513931,0.5395035180188832,0.15810180803519192,1 +-1.239517116221022,2.0273117274564116,1.144202317662058,0.4372500394976109,0.26772570267282103,0 +-0.13416702072521702,-1.6625681783400759,-0.13785468979670368,1.7714761753858195,0.6499452817249775,1 +-3.029920168534784,-3.865989091891469,-1.5086129953495129,1.3146332915290695,2.0983077881640835,1 +0.366847266002141,0.21570970400004896,-0.12294330117211535,-0.4038148022955381,-0.06551930404120954,0 +0.5263182793485846,1.7378552582708982,0.7733328770619211,0.4588370587988009,0.2717637570064002,0 +-1.006436485878652,0.5958623824296236,1.522909779824074,2.009814501178022,-0.39458277414584364,1 +-0.23785295075569834,0.2745226591539328,0.33446171382180206,0.04969838165357576,-0.3683639269488109,0 +-1.57562170005492,1.1148336279143667,2.0825938332230045,1.4655149355449562,-1.5848387194592077,0 +-1.3686914837891124,0.08413114714779446,2.9594908388160412,3.2079567362267905,-2.8085345914568887,0 +1.9388277077969596,1.8080121199268295,1.4018557474566655,0.43642293344445904,-1.6185581583293556,1 +-0.2708024291048974,-1.213399179913591,1.75252446055278,3.560176669101945,-1.2818059126463028,1 +-2.83751572347727,0.5065305009713786,1.517415225852517,2.426016438851642,0.6012173048097238,1 +-1.4039270307489633,2.3772133357208265,1.5812460746061965,1.5608362608074482,0.8942563671826632,1 +-0.46983663728339675,0.20446418969667213,0.6295149848707967,1.4170033111661324,0.4585698180944332,1 +-1.1411326299750226,-1.0808095794249435,-0.9303719253210597,-0.4315872874466502,0.9884337311114666,1 +-0.3864983747171956,-0.5663222884249788,1.9853803566827273,3.310137582329977,-1.4114100182911677,1 +-1.3653385563090459,1.121187276429203,-0.7646720165500314,-0.620235347764583,2.6777292333399902,1 +0.5348277134206394,1.5729151359933562,0.930634553261211,0.2003244107838632,-0.5470469013841766,1 +1.2402005159207563,0.5404193826508673,1.7798601002124381,1.8210123050195017,-2.009448227415393,0 +-1.484128683651384,1.175047696965426,2.1018729364928053,2.950927145896933,0.035776169970592187,1 +-4.110247628914226,-2.3432613721973667,-1.7986741668910344,-0.05126469120839272,3.1576894524271553,1 +1.7239057056073237,1.7707256742112425,0.14308125065079658,-0.21159368926852284,0.6308496017019622,0 +-0.005717840815205877,0.1663183298899047,0.9489920933045253,1.6676758429566478,-0.20250952103656872,0 +-0.13540959899611626,0.09491482743305957,0.007346254692475712,-0.18485663047191414,-0.08009521221481708,1 +-0.06901089785943548,-1.7301495402931586,-0.2611341086561766,1.371908794248256,0.40711228185221127,1 +-3.0652002456925587,-3.0243078532289234,-0.9707380976453992,2.1923744344681415,2.669187488028802,1 +0.47733585598069317,1.2117897339057686,1.404160816261035,1.1334296653315266,-0.9637575358569638,0 +-0.9080325270538138,1.784936004622983,0.7807643141990306,0.8063399236328678,1.1655004270519547,1 +0.8296511948632872,1.013896243209183,0.25928354286151145,-0.397324083527707,-0.30268811666561235,1 +4.149420612171177,3.724728747712782,2.2135084047666047,1.5782832038783856,-1.0694551081379098,0 +0.8792426337044599,0.666226960430528,0.2679966816611035,-0.027296512565038844,-0.2806108575003712,0 +-0.8328459519825343,0.25349701430025795,1.3443952031125588,1.589017904677461,-0.8441197646879243,0 +-2.276993236581929,3.880310067429613,2.021651267086897,0.12570973858639822,0.08889954728029092,0 +-0.37285755842794366,1.0369447784594232,0.22645236997345336,-0.6425247340471605,-0.06985725800170972,0 +1.6616286481781595,0.49289547788000054,1.0144888580934515,0.9224387719484155,-1.4075002055814465,0 +-1.353009858444575,-0.22968884698665737,-0.5067953535250777,-0.4871118206581573,0.8661589063174681,1 +-2.144018488267287,-2.204117341802065,-1.0208419892602494,1.23357166042503,2.2430342381306883,1 +0.6673397714009341,-2.3282291245851643,-0.2174283017642718,0.17594508510677576,-1.858131437228015,0 +0.40411269970304664,-0.04680022081142288,1.0830303280247255,1.032813003451691,-1.5645698268524086,0 +-1.7938962573228752,3.5708911598147495,2.94871873694966,2.745099082973759,0.3446045158940314,1 +-1.7207724052183861,-2.368476485800663,-1.8032892858692982,-0.39838716469704827,1.959346429669739,1 +-1.7565748774173446,-0.4720473011661368,3.1758802611569665,3.7593278419092133,-3.1321387853689777,0 +0.17188927552942923,3.2868223972253383,0.3476785525345107,-0.5264724390427524,1.8485013113517872,1 +-0.0026480153287329333,1.2714052342773883,0.2689148759104517,0.27650568643253226,0.9542753771733583,1 +-0.19396964661345129,0.3168745896549272,0.40505654736197344,0.7903348959346863,0.3107623561309576,1 +-1.0001896419257839,2.236371836225418,0.5631113375337815,0.05560858335760965,1.3285327251980732,1 +-0.17693960492790461,0.4134706102666038,0.6163002737204041,0.4758005374396731,-0.4359197696603946,0 +-0.5160561996653825,2.861890346522963,0.7842422319410532,-0.4188783331753858,0.7562134462293799,1 +1.5187962406718318,2.3028500427631395,1.3100222415026537,0.748354312045072,-0.4244844325378191,0 +0.7195649770877928,3.499969575826726,2.3969252103215752,1.642469893865924,-0.500228403899468,1 +1.1719355838562509,1.7624036656846473,2.1405424141024536,1.7930458818060795,-1.631479405066993,0 +-1.537155653914023,-1.1430079975241618,2.5640519702804325,2.9285498817744973,-3.3698980718380476,0 +-1.3303668461572835,-1.6936655415019657,0.1773437990241673,2.992470708418673,1.6313886235603063,1 +-0.11345100289142196,1.2757756482670777,0.47708061662551604,0.43195930792105586,0.6827564553072836,1 +-0.7629179949483428,-0.27617618588886494,1.6418570359774303,2.065451141500074,-1.56585039423056,0 +0.27381437870660386,-0.9383735463170196,0.6993792590251338,2.169402481165384,-0.2704310280433495,1 +-0.9791622407826074,0.270416996696901,2.0304469526919258,1.8343277353937997,-2.1039083332238695,0 +-0.49317933220354093,-0.469908183137298,-2.311557095863225,-1.8812150755024732,3.000144029002528,0 +-1.1583689891285105,2.484658669196411,1.3413636594175335,0.4204057953671425,0.2217444390214922,0 +-1.328374813979823,2.054148338194013,0.6209088244945542,0.5709631385616609,1.6890319907435074,1 +-1.4433267867550246,0.5389333818433084,0.05596400460981754,0.8055529275293867,1.7813067780769112,1 +1.1255558323243133,0.8713903448230291,2.5295266785617008,2.5832227537339847,-2.543809498996042,0 +-1.9077120184865701,2.012656231215293,1.8963976913128644,0.46297139305479584,-1.244343659147184,0 +1.7140231039399492,0.5071868918665747,1.2425120495144975,1.9481669312034033,-0.8116313158407906,1 +0.10765372117260164,0.008477535250883017,0.4969757206503288,0.9564618719748301,-0.13044379468479173,0 +0.43623422741147366,-0.993439702588401,0.36420887663004015,1.012663057589386,-0.874247679269517,0 +-0.38776382127101605,-0.08182069451372143,1.0540803991215468,0.7801751938791066,-1.5449116845723283,0 +-2.7982373866090287,1.2723319992175555,1.8145640403062162,2.4480966048395794,0.6887414382239595,1 +-1.0098201002277636,-1.1315710190249835,0.06808451769622126,1.5459025073677108,0.7484647736159963,1 +-1.8012581917965789,-1.2400219157024002,-0.21713129920110785,1.5226413466107764,1.5431235820133866,1 +-1.1843282550932224,1.177110446385258,2.189625540017001,2.6122227628217933,-0.6398582723060069,0 +-3.0425313386137356,3.4612277851324533,2.9673163602028954,0.64649521053878,-1.7008058713052556,0 +-0.9501769856512048,2.6863447252824826,1.4481122133444377,0.16095680400245038,-0.17996733110846885,1 +0.0070065087021962125,0.30716593329307795,1.6659192709592494,1.5951034761800502,-1.8133120082616032,0 +1.2822329830386654,0.09144681253352793,0.3460389984813706,0.18807937256998708,-0.9363283648253663,0 +-0.5716175052931314,1.5781398662006467,3.0564030733390273,3.026876547086267,-2.003161067061893,0 +-0.9026113290819358,-0.5224259307966588,2.4894046721277925,2.590331889504599,-3.1594808993814048,0 +1.5829937155239604,-1.2071240414216056,1.6532662227900978,2.1237543707526227,-3.2483026275832207,0 +-2.3680471278885697,-2.15684179236145,-0.7741246549537151,1.500486331966433,2.0855669633490037,1 +-1.6838723397919555,-0.08734707306304179,-0.4801719039874643,-0.3303521632621904,1.2305254383673982,1 +0.801379885979252,-2.6351654150201567,-0.19510740518740066,2.3905288601758707,0.17609290414542078,1 +-2.532040105668484,-0.5900008465689135,-1.6179032711574552,-1.935003527417177,1.8907936297604286,1 +0.1456884132478502,2.0820787964025005,-0.3505622039316928,-1.0233533274210358,1.7267885386514572,0 +-1.471730687738439,0.7042799057666588,-0.28328931932744356,0.1222656291738562,1.993327833140542,1 +-0.21877728727881007,0.3273165792722388,1.6698874657906624,1.621856038253389,-1.6971449796815496,0 +0.19833479844033852,-0.05105565055445194,-2.1345001702733297,-2.126707512408098,2.505803869865361,1 +-0.3384918033439622,-3.1185950322221307,-1.219869903917575,0.9442271183634823,0.8650616053327775,1 +-2.216001962437797,0.3425060939229678,-1.0097165848325518,-1.6164461912252541,1.6555064522604797,1 +-1.645547608357818,0.18892637083347075,2.126344615203061,1.9906651898105288,-2.0129830616892694,0 +0.3198807491168083,0.271214432333138,-0.5313207139639615,-1.5317303164853517,-0.2862159027618647,1 +-1.3305257118168767,1.0000474937160033,1.8638032194058574,1.008858882677166,-1.7760290437478667,0 +-0.14186246014788073,1.9380524371343952,-0.1676419749266197,-1.2526898465888396,1.0007870669823262,0 +-0.7777906816441998,1.5216542476681885,1.1367184761832103,0.06482971498336942,-0.7867490307177385,0 +-1.3711113753391018,1.7012368730519483,2.9944207726069605,3.611467564510355,-0.8236150646458953,1 +-3.5698645750027986,1.281549646194884,0.801958064589998,1.2375772796196842,1.9793921669505454,1 +-1.705554974115564,0.14213596459546918,1.3680483221672595,2.025481914233727,-0.2373760629193904,1 +-0.41122809910499314,0.7656073423046565,2.281126834239826,4.085357506549949,0.09772823370246442,0 +1.776950520686271,1.1707804561648383,-0.35934093414156837,-0.9539216447576471,0.3629776349908491,0 +-0.6797821659099381,2.9728826240702846,0.27980648073495445,-0.11397563857829862,2.43164421738636,1 +0.9792612862201231,2.3096035037145723,-0.6623856874455953,-1.1767534758755054,2.2317142101811447,1 +-0.5194548731507658,-1.8611682968692314,1.0344207091386592,4.218275539692498,0.5475366750097552,1 +-2.547647060850711,-1.4749665423136045,-0.8816750506890066,-0.0958061576013649,1.321626529009071,1 +-1.1494131177027485,0.05176240585600911,0.5166794385294907,0.9909250748592698,0.3267376207357362,1 +-1.6090392983115651,0.9445539263957552,1.776702778069462,1.9933617647175557,-0.45143251943709495,1 +1.9218431160822096,1.2803774939366295,1.4005179822140659,1.0883824546119931,-1.4206909464193749,1 +-1.18971013985211,1.494949382889423,1.4631918883697768,0.5517447149996417,-0.8986128261731343,0 +-1.5992718521471114,1.4518238386046303,1.6768510240317278,0.9806251193521486,-0.8291957705924676,0 +-1.5038643270244751,1.236400746630434,1.7836141643376533,0.4544378451425658,-1.9049659642393242,0 +-1.2065399080253272,1.6027538155903573,1.0338117032374574,-0.09215526864833201,-0.4951162750254262,0 +-1.18419496610267,1.0332346012095546,1.0969932817834698,1.9412946918507852,1.0188329613847114,1 +0.33204437903351525,0.5488852674267048,0.5797586624972211,1.3117446030022313,0.5346678691489415,0 +2.5012255989618417,0.508314601057178,2.186470226717562,2.344546201369118,-2.8340036227623786,0 +0.46119749083218653,2.586734150636205,2.8072378030952096,2.6094266849386427,-1.2186632065343408,0 +0.15613018524727362,1.2140740460768482,-0.4890824830250456,-0.8372870043881373,1.3803351063559552,1 +0.24789232426352537,1.7932907754121028,-2.0335588044533255,-2.8515779784766613,3.3048935105273314,1 +2.6434253734868594,1.980239090321485,-0.7290186451547664,-1.2927668285242668,1.3705709464840177,0 +-1.5100594086482027,1.5047071659604536,1.7465651350606572,1.094275146527267,-0.8431321559064526,0 +-0.10951270911021715,-0.03825472735575608,1.0300132019037473,1.5030016545346672,-0.7425850301829096,0 +-0.9864635572748018,1.7791207375463336,0.20430972510145423,0.12913464649312778,1.7812321756445741,1 +-1.7188054836648354,2.1269667901634026,2.179443862242762,2.5361067603036247,0.43143923343390034,1 +0.9213887048698323,0.4465927587684355,0.32392015536577046,0.09516809636412182,-0.5101450475171183,0 +2.769601769634144,0.25916500307055035,0.9485429107607269,1.004721440840574,-1.7694340236915853,0 +1.8591468380285128,1.8374819247408811,0.9557563786305253,-0.3378606825209318,-1.3771915083804727,1 +1.8448906938165135,0.7661158425755981,1.3340342869675363,1.197776640723994,-1.6352472234745952,0 +2.6650337513580835,1.3906442223205389,0.43491726714576057,0.022992808719987434,-0.48641258096136697,0 +0.30580303266224806,-1.2909577874562497,0.8462558662407612,1.560450511198935,-1.6468799862779573,0 +-1.6588095901314563,-0.7850500219958799,-1.2557530117488673,-1.2267507307486896,1.3398326939895422,1 +-1.6345041678711967,3.7197854847320793,1.0076916637382267,0.12691358336933145,2.0725878172571806,1 +-0.4559165174615557,3.4709742535130954,2.556977707536956,1.9739990295687497,-0.14222390844416632,1 +-0.8071537472670868,2.374192210362553,1.334355834710894,0.36670225194067285,-0.049639837090545424,0 +-0.058383333399446236,0.5399739059782869,1.1375084054926567,1.6389524677538017,-0.2804674145368551,1 +-0.3606607538960115,-0.4527521623868842,1.6396495464047507,2.140515534208917,-1.7899879916979002,0 +-1.6911275743002423,0.3559282749924805,2.411576223272971,1.9782777314026871,-2.507470447506396,0 +-0.8305961281561369,-0.3744856780445509,0.39856041310159307,1.2707529204521435,0.3754567960985222,1 +2.2687351329289687,2.9830955012692995,2.56097721997783,2.207801091707426,-1.2970819137824918,0 +-1.267509822926727,0.23678705563230817,0.9096634785153505,1.5149322583130218,0.21467468635548947,1 +-1.7353818017447797,2.547248621361577,1.6823822855515067,0.29899906803575427,-0.4489748013289787,1 +0.9334768437079146,-0.16851626636402983,-0.24578963088546169,0.4354525384328032,0.5692703128165573,0 +-1.4938324116807502,-0.15268610379923309,1.9307549161137634,1.6601758136830027,-2.3152552177458134,0 +-0.4502228007340068,-1.2716996565842793,1.3079613352644621,1.512240346924726,-2.501135566776727,0 +1.324980961125342,-0.3401760256146382,0.8836305073645911,1.7043861660027737,-0.9641224692528512,1 +-0.7690063883819765,-0.07973281674275512,0.28288231158829635,0.8891136972674495,0.49908782382996386,1 +0.16922253367772466,1.5810197703323663,2.3019547287415922,2.184329532252888,-1.4215444310975924,0 +1.5917829538733783,2.8858079265047962,1.6253992315567352,1.027466688378091,-0.2909022545011075,0 +0.8523387819187136,-1.9320318201107458,0.433837586427528,3.200783971330458,0.29298608112612257,1 +3.327346855388015,0.8951432849423449,1.6088840668496356,1.4983745743433265,-2.3115494378658754,0 +1.7559324372435539,0.9897838443788548,1.6111419431509204,1.4598757906039657,-1.7372062358021596,0 +-1.9735935411217058,0.8749862984359823,0.6368795555971938,1.0547161003369563,1.2190385664452061,1 +-2.701054660433766,5.90959121657059,1.9476797271492183,-1.3881667994873026,0.7702608282414367,0 +-0.9149693927528287,2.688451944110236,3.1941438773235964,3.041802086972155,-1.078989477916013,1 +-0.06018688702356534,1.2908047297399743,0.4782912825262234,-0.04693557786941449,0.14975977877073077,0 +-1.8555944689443753,3.60961218727843,2.2715542927827657,2.0126809111764254,1.1728475886609593,1 +-0.42801957388106504,-0.06948587306147358,1.7706080529346875,1.911260563885678,-1.9403188502360262,0 +-2.1596784213490743,1.2788453783130547,2.595165356307195,1.5621503652583373,-2.310498862214711,0 +0.9414855632001368,0.9227114717473404,1.042264662311528,0.744181803804657,-0.9959987974590196,0 +1.6488479139843257,0.9428412154598211,2.134451713722145,2.211941826930293,-2.137521220187744,0 +-1.2000326867057887,-2.1077527197382775,-0.5328093517240292,1.0373935968872638,0.6716165661072553,1 +-0.8084390039700073,-1.4165174105694187,1.477978242515306,1.892622537003421,-2.502761870717478,0 +0.15187062556406405,1.9064125314524154,2.371472348485823,2.066468618167821,-1.3810601298145497,0 +-1.7919595850441108,1.103751066073377,2.4063460159007777,1.6896261408035547,-2.0297291696701576,0 +-1.9175753084317257,-1.23788015287841,-0.46782512953916533,0.5276167356473772,1.0717211672524734,0 +-1.488633857368394,-0.054646451880202895,0.04944845900235584,1.0186175727199696,1.4512868745224219,1 +0.9630443137692635,-1.6655367739939968,-0.6051843247798073,0.07372315892581582,-0.5035609691525107,1 +-1.0481334991429372,0.9949073653800542,1.5551344233455768,1.2694636619927464,-0.8708579485147733,0 +0.24698901691181208,3.4943141281505934,2.9529559016225866,2.3476867451124566,-0.8642658577880722,1 +0.2161563918790428,2.232362506731066,0.42817327737166155,-0.08800682845512564,1.072250189886585,1 +-1.3444934432122482,-2.3891419098954243,-1.2638915660093333,0.6026356912405605,1.6599656064849952,1 +0.024415716168932455,-0.5688909420701944,1.136592620200569,1.4477804732226938,-1.628175495545304,0 +-0.8760802559766744,-0.8858958942335887,0.7415445235893364,3.017439355329094,1.0037023809512002,1 +1.4970206461238103,-0.17924232453956623,1.7480736737991331,1.9198234729668364,-2.6342334414992377,0 +-2.9025732974130434,-2.1337508124847218,-2.1713698734025444,-1.6794373801209845,2.036596142332521,1 +3.276812503962505,3.84545310187302,1.4819977972932543,0.8001318903547774,0.1890681655749935,0 +0.6766897858461342,1.404571878480432,1.729646874488859,1.5059304636317252,-1.1843546640647629,0 +-0.7135573328012883,0.5311222557399757,0.9462466516077931,1.452088641968583,0.16958854211931151,1 +-1.3972589585427044,1.472933388058028,-0.35682144099468693,0.08702550420154453,2.8711042222151395,1 +0.6906815447632689,0.7094265813605418,-0.9940578954910266,-1.6290591370732659,0.9979612167252345,0 +-1.7793054748284316,1.6704589375456038,-0.45349722795737446,-0.6935535354617093,2.562555157686379,1 +-1.1615026975667586,2.0879940663508916,1.4943596157311,0.4673708277793055,-0.47902276255846843,0 +2.3621090732510632,2.9345721896907975,2.1747106805848198,1.7372013368390187,-0.996708123507243,1 +-1.773035253504721,-0.3358926167437134,-0.4630227180499953,-0.2896108939917339,1.016261529852953,1 +0.21572857739644924,2.1849131351179976,1.080607699196841,0.5072129426370945,0.16253912483845467,0 +-0.5891073325387702,-0.5782993668603442,0.5129727325116784,2.182383934423773,0.8279108701518112,1 +-0.2517475870977097,-1.690790551283349,0.43106230786184474,0.6743782165073311,-1.8705545476856598,1 +0.8902195738983008,1.29733536397171,-0.7571090998042564,-1.2212095448940619,1.4182748479696587,1 +-1.1415177317497691,-1.4521454254737503,0.6297561398641274,2.0888949897271205,-0.23736735681445098,1 +-2.7375429020405004,-1.2065243656019469,-0.13881985429718968,1.483703793016306,1.665717075065149,1 +-0.11182890040764404,2.063950330835677,2.8006501966041695,2.647544230590724,-1.4931866959602722,0 +-0.5582241182706771,-0.6012587412974444,1.9920417278712246,2.91430923374521,-1.8402247431968004,0 +0.5472307271891212,0.5897747826491881,-1.002535729832236,-1.479547706911628,1.1105110842520363,0 +-1.5764170017068844,1.5642035901610654,0.18269992603839527,0.1892190801545801,1.880132773231884,1 +-0.5308057222213234,-1.164979639692924,0.05428854092936475,1.400273435308196,0.42601376650224676,1 +-0.6907888013513492,1.1533964105469106,0.8206580601626711,-0.24810467652423096,-0.7944373699465893,0 +0.11330849715657765,-0.27108234923781116,0.6243441834275627,0.633700775598711,-1.0643116796696994,0 +-2.0969545097883238,2.9995467569782326,2.414794873631009,0.8315396848223107,-0.9916690303296287,0 +3.160678770215427,-0.3479078561204141,-0.08015416312560203,-0.17058829255971242,-1.4103756103197547,0 +0.3557135510383428,-0.1021811014568863,1.538804294177217,1.8323536772689688,-1.7836382873168624,0 +1.523679083144863,1.7032106974118095,1.9455119996357726,1.7826254610201802,-1.366578072510832,1 +-0.443694302053508,0.3199184681264716,2.2318727346012714,2.3141032710516676,-2.174140228339026,0 +0.2267364292742684,1.1204893029538288,0.5631513340730683,0.34899692435067053,0.1213920464084447,1 +-0.7138972900348539,-0.5188473758238259,1.6653338683950802,1.9963663403407619,-1.956243858790078,0 +-0.41008006659012,-0.22067762119657863,-0.19321914934776818,0.4849390561182194,0.8998738142180547,1 +2.3528239387054635,1.447600522852698,1.4958458577447777,1.1658583432655234,-1.5340788255919944,0 +-0.9698667180222789,2.519745420244412,0.883880433755537,0.7587356967071283,1.6305182557829405,1 +-0.8408049180962189,1.3073445737517169,1.037952645082397,0.3097066135825779,-0.48100823703774787,0 +-0.8754693856176908,2.6144944385762487,1.5782160705354282,1.7486608359493474,1.1687295719753499,1 +2.1083816800219592,2.614232609896483,3.1398907182640823,3.1124784369710365,-1.9635145932819413,0 +0.7236668628225562,0.1683242179854031,2.247849000252773,2.477760657192853,-2.5746253914494406,0 +-1.8109867070276398,-1.891841329052113,-1.0640913632434545,0.3195167719960087,1.5383667183523715,1 +-1.4296093594225097,-1.7531032446502963,0.37878855241423315,0.39386110364301596,-1.7252206389205746,0 +-0.456303082176031,-0.8915445608665182,1.2936445067970495,2.291116953186697,-1.226988656490396,0 +-0.66278643585668,0.6162581090701533,-0.41110134218053124,0.0386615153977421,1.8390007432939135,1 +0.7487735326691423,-0.95651138909732,-1.2779626285674044,-0.05467088294786904,1.702754091618667,1 +-0.5236373291767441,0.7542031434504037,-0.07430606922740796,0.02230951134950876,1.1291004878836166,1 +-3.288836655316611,-0.581264002887689,-0.6467283348076541,0.48363778029833937,2.557657695483859,1 +-1.570831566728609,-1.7342633401406062,-0.39366093395374935,1.506131601998484,1.3628784660165416,1 +-1.7580967114262767,-0.671632653808919,0.699097378318631,2.8393999010237976,1.4170316738424087,1 +-0.2774936151877422,3.1954625795441616,0.8171782375525725,0.17191412238983284,1.5845498759804093,0 +-1.1322226060679792,0.5506358768789636,-0.5894208340290353,-0.19238780464665006,2.091077722738583,1 +-0.5966721610670477,1.7330820255289945,1.7347324051017243,1.909224209527983,0.0043833089628941835,1 +0.5307883415429602,-0.6785132068020279,0.10520625303170589,0.72185228667986,-0.3080095486977944,0 +-0.38788224291479656,0.31139992454797616,0.8671257597772843,0.7527755487770244,-0.7459609166355752,0 +0.06800799056882667,0.4514059624425565,2.0716078647052183,2.097063519806999,-2.080169554557494,0 +0.98511330861818,-1.5219536295651173,0.6483100214791248,2.905291898718803,-0.16482507693906068,1 +3.1560749763335956,3.126055185386031,1.8837236250112832,1.3791257417722829,-0.7884941256972091,0 +-1.0175436476711797,1.6804446353937283,1.3779747352861762,0.5930051402724767,-0.5267429234359844,0 +1.4444090737397222,0.04829045586017933,1.8050646979722302,3.5113456159708054,-0.7684347740987922,1 +-1.4170234162668867,-1.6921268815511263,-1.4085582428228889,-0.48725582650640265,1.5192988686979367,1 +-2.686693216551752,-2.678464935177344,-1.4861200605947151,0.648800566802079,2.38799282314348,1 +-0.11602321598477128,3.858555785702089,1.715166513580368,-0.6198092205992156,-0.7658084202591131,1 +2.7004629848254025,0.8565439076770457,2.5497831912264033,2.679762800970463,-3.027796143696498,0 +-2.257222392597491,-0.2730152242941035,1.554985203118055,2.735052219313946,-0.12194112971279591,1 +0.8766567725406371,-0.9074525321093523,-0.7615188272898079,0.23089505135428678,0.8220968556120742,1 +2.0068324766898495,0.5536897626235654,1.8189768655657037,1.8060815107932464,-2.3607493865070532,0 +-1.2199328094602708,0.4566426580298767,-0.9148930858857474,-0.7143674280298509,2.2086072993981625,1 +-1.0545491306018464,-1.1667150035352236,0.9956658649604131,1.3299078148463799,-1.6674723951878199,0 +-0.1395757973777425,2.643555230869384,-0.2331323036822578,-1.8290986863598513,1.224810058312603,0 +1.2453414026136969,-0.46462281614243683,-0.7763111001506413,-0.24726982076077375,0.6501575297356373,1 +-3.640081404011381,-0.11990447539307736,0.5178899796948309,1.3925893754311534,1.4296526203456632,1 +-1.3493959574253283,0.9858507105031514,1.7537444521891752,0.8426300587819658,-1.7109561571026615,0 +-0.19083777976506122,0.6065206600018778,0.9049000746692073,0.8475501532134864,-0.49973195884690724,0 +-1.5632796454469933,0.7416870801582529,1.5981424841314364,0.8506444019953047,-1.513156443324729,0 +-2.153588058004878,0.7796355755353925,1.0804664822193728,1.652186668096935,0.8098701445318062,1 +0.9959353011812602,1.449443322428431,0.927673990140043,1.2814710464711445,0.3719968942351898,1 +-1.4182143304775856,1.0469901609519427,2.5191864522352105,2.5427278315498008,-1.5350308127499308,0 +-0.876735111102997,-2.110427240389935,-0.602463587409089,1.3378372176229663,1.053385622746398,1 +-0.6529429564127256,-1.1608948988120233,0.9258363336804656,1.131039147894836,-1.8529115526360482,0 +1.3990084229251694,1.5342844167942502,2.1015267742118295,2.389986822018118,-1.1881806875867937,1 +0.6808124881003792,1.0414607269251672,0.2962953646541774,0.528439971746496,0.7082574972726804,1 +3.4554202561303633,1.6959212064291327,-0.7959520694438482,-1.57219782531345,0.661175761561658,1 +3.8665136782701204,-2.044842175492316,1.3452472104021156,4.686602434062745,-1.315173426145115,1 +1.5494152436192974,-0.11365639202698308,0.5913889829009933,0.5465279346165399,-1.4075753654175231,0 +-0.6030418957453034,0.19944836600944682,1.156544862701827,1.161503766555847,-1.009181303825639,0 +-0.76681573788926,1.5866540820454882,1.0593898672808926,0.42683791514480096,-0.14672491419122735,0 +-0.3910275518201166,0.19659793844158002,1.086817972229217,1.3815441255620944,-0.6786334497093859,0 +-0.9265849202018711,0.7980436920559595,2.1604304990274366,3.526166837959819,-0.031908363214988356,1 +3.1127131657114493,4.277257032078616,2.93207409161902,2.399106326355807,-0.935526208049536,0 +0.5251160974614526,2.455560570539161,1.8860235015645155,1.5074314907937916,-0.44211702886658866,0 +-2.8938234787344186,1.1847271095538785,2.4596198177862814,3.0239814767317106,-0.23342883165350115,1 +-0.036603873707149126,-0.08978642912210888,0.5868494442657264,0.952035694424438,-0.3945033655008592,0 +0.0908531621410702,2.725352131469817,1.4443506367230021,0.9309329596276316,0.3666049688839732,0 +-2.040782595342465,-1.2618020112958603,1.730672395492789,5.722230540968317,1.6951819509458008,1 +-1.7826215279889261,2.411124744932276,1.6644222323436262,0.24283444944383492,-0.5896503709066165,0 +-1.0847774876769953,-0.4981208393778691,0.6258792392720406,1.9793400645040389,0.5882961348450608,1 +-0.5248498792555738,-3.4254630303462603,-1.7781575761409902,0.411094759636623,1.3316154541210494,1 +2.6068427658039663,2.018125509836623,1.3512424122575768,1.0373640488612226,-0.8524973684407908,0 +-1.7836452834718393,2.663678230605787,1.6562758374397533,1.5233837446424625,1.0929990063291457,1 +-3.4897534428022814,0.6808892475210913,1.7952948668635655,2.7275243174189505,0.6803821972080282,1 +-1.3803238098579627,1.9574131869020426,2.013724069411576,0.8353446189270175,-1.3394959188993578,0 +-1.1939037576484306,0.7937519701457905,0.4234457433310308,0.7643150189376775,1.0527790371393708,1 +-1.3093216465162638,-1.7732816494193844,0.08282481558493404,2.8878431969154175,1.6491824072614776,0 +-3.155228856947394,1.9764146706475758,1.1225602319490682,0.8160525335239222,1.3268213574617478,1 +-0.511780820057914,0.20464328232417683,0.10940965251305812,0.6552620368281307,0.843890208535902,1 +-1.5235159845879518,-0.986812378380802,-0.6675434683658615,0.38794739418293966,1.5020079202734378,1 +-0.13039556802590058,1.0647890814253471,-1.5126359344858669,-1.3884417967681308,3.1009623936536013,1 +-3.3860883509310513,1.093529664186957,1.5067743442413417,2.2024906746957145,1.152245915537767,1 +-1.858085434430539,0.31162720992352744,2.744515507535854,2.5915583067675727,-2.5950268654947384,0 +-0.028966541220662334,-0.66246243969678,-1.2206923860911534,-0.7659900190170226,1.3418387635169227,1 +-0.5255570855869238,-1.2759556559746237,1.6637387449055816,1.948026997774746,-2.827877342868727,0 +-1.4940387174298784,-0.07812800183101687,-0.7869899489637688,-0.15878544179582277,2.078780937957426,1 +-0.8708683955243985,-0.5148833170873108,2.425100453500221,2.71109037101316,-2.87999184818086,0 +1.3820967540181592,-0.0384000208125459,0.9404258842605887,1.513196123173913,-1.023448682742154,1 +-1.8400039947331956,3.41720808117424,2.8841704301951063,2.534845370215021,0.12463772508150395,1 +-0.7679815270160875,1.374659791583743,0.6441817141129723,0.5609736576504182,0.7547833721538099,1 +1.5502728560130177,3.5151776227337694,-0.9334955905080059,-1.927835145711358,3.05221786080825,0 +-1.0133976684445691,-1.855639883286227,-1.2747199766364345,0.18755074943697325,1.6517711049969923,1 +-1.239807076172337,-0.22556731119382278,2.193866273719335,2.1271557946394806,-2.571347534830145,0 +-1.0497735623524769,0.247319707162016,1.8318721304522931,1.8986609604660583,-1.570580245815047,0 +-0.6339883801771671,-1.9146089350225837,0.1356181384856072,0.524507799177067,-1.4444374016826966,0 +-1.0912524439034992,0.2707191822792726,0.14815714780438471,-0.03176829859350527,0.25778176049686197,0 +0.13316114372665955,1.7030853775952708,2.7084748047967007,2.6938057970569473,-1.6717276968595385,0 +-0.891771641065202,0.983554766350477,1.141386508387345,1.0672193660649836,-0.19489851689757443,0 +-0.4897488164627559,0.9382719473258937,0.9092157469149597,0.20771416765288964,-0.7815656123062925,0 +0.11602283678192371,0.03852026260813224,-1.0618618882344006,-1.505529142307966,0.811974261123954,0 +0.17158098962972723,1.1950474927587136,1.1133605011396646,0.9987817277995561,-0.34976143618156774,0 +-1.0372066603137116,2.053832240738945,1.1966146651218597,-0.6463644879956312,-1.0888018943553013,0 +-2.341385680295058,-2.94516533941347,-0.6555431281411428,2.8588393728396557,2.5064145398475928,1 +0.4271245540015699,1.862571998154856,2.0063998559924503,1.8059528039618238,-0.954954251164607,0 +0.7083984253481592,1.723196084481783,1.579087277008166,1.4641590309694257,-0.57134963701187,1 +1.1071903088718493,1.3254449036234064,2.0068424975336323,1.8712951164992195,-1.650572969176435,0 +-1.5950156404697586,0.1518178979248107,2.032831028401421,2.0467381593890392,-1.7878393921093156,0 +-1.153290167760378,0.8996108601316863,2.2389567499614937,2.018558520027241,-1.696965512875964,0 +0.6301771672888119,0.5036283089102747,0.37958813535872377,0.24612510632011986,-0.3185480448596718,1 +-1.1856054400060805,-0.19328275201263567,-1.752871875935768,-1.127766317156698,3.0397551824193445,1 +-0.2647356337079323,0.45934449712542824,0.7237168699687244,0.8428415441660841,-0.2061694839032533,1 +-1.8675738663873889,1.4602879908382103,0.41882358349927695,0.19834591387549128,1.3343856624852122,1 +-1.5831967483394171,-0.21754303143273734,2.2150056967825438,2.1409498631829798,-2.482101949340053,0 +-3.6831488583181065,-0.8419704113106123,0.49093972074233183,2.05780331485789,1.5155483865760835,1 +0.4428756247269867,1.642005742950141,0.27362896621613264,0.4796798257688849,1.3890974763219601,1 +-0.28247435788970854,1.3790458798442204,1.353648274181296,1.8261274356087134,0.33911155268972815,1 +-1.6573393344967426,-2.3720569139947703,-0.7465320349367637,1.157968565362729,1.1900459001620487,1 +2.174843149496501,1.2456309040933018,1.4320801023695466,1.8305273424236708,-0.7965325878682232,1 +0.7406299668606107,3.932054018809741,2.9213796216874437,2.278736120820226,-0.5935350568181474,1 +-0.12019434429737885,-0.5097526528185907,0.7732558229197607,3.178200213703147,1.2304596762652156,1 +1.7337767175294372,2.3290760578120073,1.8801083886930934,1.3634508773440597,-1.119142689366958,0 +-2.444520399945463,2.5132380973507087,2.526128098224236,0.2700893174138893,-2.239849022074524,1 +-4.70519490090936,3.2492453248459365,4.2564335763963586,4.628980947114554,0.031208998567317625,1 +-1.919551280266035,-1.6453047505527714,-0.6631506675782501,0.7495000373525909,1.3627138110337107,1 +-1.1468513927273265,-0.2133287920030208,-0.29695435707432677,-0.14941715117483856,0.6971293391478749,1 +1.1367728545851543,0.9175568488168211,1.7466491698199405,1.7278272296270771,-1.6220673407996582,0 +1.673031335122776,1.9820458510016563,1.769115188147199,1.461129373959288,-1.080859469358661,0 +0.1807265107066609,0.02793496612491364,-1.1491126072671978,-0.8761919543569177,1.6756834906086073,1 +0.1321620563744188,0.33743983555836266,1.2732231306881592,1.2632081287175376,-1.2769078604214439,0 +1.8798256406692155,2.6530863812138006,3.2422277193376106,3.015987008182914,-2.192227951175319,0 +3.5921605710401807,0.8313301670609557,1.8514055498211393,1.9504786307942443,-2.530654061816821,0 +0.749024358376073,1.0735761130305639,0.5438720047423038,0.19580859275836593,-0.2246805197546642,0 +-0.45712891775856734,0.45288526721168276,-0.24662624451086435,-0.6747731182570087,0.43796261944830317,1 +-2.71194067566681,0.5788336594263372,0.9770255405548265,1.5962078294172504,0.9748714644047641,1 +1.2548459122000066,1.5344973018844978,-0.09540119743810749,-0.1046553333868987,1.2239469015135578,1 +-3.2090397744472408,0.8441389121874454,2.3538122086767053,3.541566202954287,0.34702734181870465,1 +-3.603602496727503,0.30204738924339203,-0.0585544492972051,0.639791879815238,2.352367745298227,1 +-0.8799527469125623,0.38184389104423133,1.9075472104449394,1.6873971814884494,-1.9013940113781829,0 +1.1536787979977339,0.08946332189273543,0.38311479716547914,0.6813398530821406,-0.43824553187788773,1 +-0.589559297299004,0.3919454845694843,1.4086131057263818,1.3227845261567668,-1.2295252397089191,0 +1.349386722767997,1.2618327501407576,0.9983252674802551,0.2385546136753377,-1.2474886315229026,1 +1.942797695566953,1.9309200576425727,2.7775642977494543,2.7509480561910755,-2.148174680387887,0 +1.002046880004305,-1.892648735870731,0.023012749805542874,0.4113738696354209,-1.8338560526355887,0 +1.7813510200730116,-1.6546684205652724,1.014315067932672,1.3075092018585446,-3.175941418004679,0 +-0.5618537732684308,1.618488805872934,1.012725910739245,0.1447237547702669,-0.38568721169161724,0 +-2.080050480489361,1.5660145076718206,2.8659384999119433,2.3994444192684,-1.7571911585377875,0 +2.071861459031807,1.5819321915413078,0.6842112988791422,0.425860265096731,-0.23197035572082247,0 +0.3782599074721582,-2.078780642357663,-0.7227436643662539,0.9848746921891894,0.5552396910635458,1 +1.6106571673951882,0.8787245690993364,1.1720498431601665,1.88736571994104,-0.3075636336434311,1 +1.7265571284815788,0.6750134277959425,1.71739722520508,1.656506906441229,-2.0735271592389144,1 +-1.8571521720443225,-1.7343727603049683,-0.4627300347016521,1.5857455085718173,1.7071645081701854,1 +-1.077522991542846,-1.2900531295770343,-0.5945495204590028,0.5722603030349935,1.0815956019788695,1 +-2.0928353187159514,0.53968827405994,0.3885701807105375,1.0889614067167963,1.538290681082815,1 +-0.2368554052116022,0.763173959885151,0.2508542640940444,0.44507665567795907,0.75103474546639,1 +-0.17500975830331267,1.1948238951526324,0.6595719123685733,0.9057123596041258,0.7194421170262646,1 +-0.4880058090446492,0.10507398206579455,1.0825129833689229,0.9402872755901576,-1.2137622072325325,0 +-0.5482310609281269,2.5652066694887883,0.43834301824644467,-1.118621033240324,0.5036946977706835,0 +-0.7432873861688771,1.3860043135747833,1.0781373247593005,1.2609407906680994,0.5191844105891736,1 +0.46003939824180007,0.06527853968219599,1.7758095117359025,3.922472647043912,0.09960297978682775,0 +2.0566995922923157,2.8233379447604063,2.5638305003110267,2.12815830733511,-1.480441725656783,0 +-0.3142431574444535,-0.05237636071455276,-0.5807639703942844,-0.9575773951581978,0.34942837549444405,1 +-1.0745076304304773,-1.4429762013880754,1.751184166261124,2.2199155219266613,-2.7151689824538043,0 +0.36485470368561446,-0.39559115176352844,1.26830637169416,2.842861075611598,-0.33874577218613355,1 +-1.2946622420459146,0.9964256947326345,-1.487761315765701,-1.1956975863252444,3.577373337472808,1 +1.0091617893408145,3.367412559926198,2.1766050802419397,1.6388950345670983,-0.22165057689568102,0 +-1.3241562711212087,-0.12930751638603732,2.5346197243227078,3.173811146929783,-2.0866434087025976,0 +-0.8829825865803116,-1.050045421655625,1.9606925364261927,2.5449369288618273,-2.514993952910323,0 +-1.2380174934570043,0.1876570920732401,0.5614162324401284,1.8227400549176533,1.30463796360309,1 +-2.3818980596917485,-0.30886327968594274,1.0235360997939398,2.089295530039154,0.4091333019294171,1 +1.5651112725571985,1.6554161546278303,0.031708271167287305,-0.5208706641715832,0.4870298102594557,1 +0.7269669778919601,-0.6639178795181104,1.3415325792426396,1.5132872963924964,-2.3637160109409985,0 +-1.4025270378771655,2.6653524268381314,1.6014118176556689,1.6487756711326103,1.2325508293181566,1 +1.6015599632242106,0.7549727558261072,-0.7852362540638603,-0.7341058545680181,1.2378402084603441,1 +-1.166657713394523,0.7611010353450893,-0.4986416985261886,-0.3542944875261367,1.9241218237289077,1 +-0.08026108827117384,-1.0551887647072387,2.2203672761710167,2.752306630379624,-3.1651413028501896,0 +-2.096113443515143,0.11368718361993846,2.590565279352208,2.210684498029421,-2.7748948516399574,1 +-0.966071036490391,1.7488895635489248,2.018014337656998,2.256454619537578,-0.13247807411153434,1 +0.12861415343437255,3.3317946794182824,2.0588839677982294,1.840466523779194,0.5338567814090793,1 +-0.5419240082311799,-0.6473695054433745,-0.22635221725592625,0.6051391061767304,0.7258271190494494,1 +-1.5296234487179072,0.969558804188375,2.6470493942589446,2.4718626262828467,-1.950766825813361,0 +-5.043892456948213,-2.74681295732993,-1.5409810893809206,0.7273363894669729,3.324302413409618,1 +-2.433732998487467,3.7642062399118905,1.9361466166729882,-0.8702094791938009,-0.8727418694585812,0 +-0.18860044091502381,2.2716896424978454,-0.03951113263532438,-0.534151969717525,1.8441161592508468,1 +-2.8199504306852257,2.4428426011811064,1.2359908030063336,0.9839361720802553,1.6029105003866508,1 +-0.1615310057905135,-1.4330769848278275,1.1737290771554452,1.8823932255726308,-2.039847744862965,0 +0.13286117321826718,0.07732528396537641,-0.8578806906879226,-1.444775925162166,0.4376029458804199,1 +-1.4466550709733417,-1.8740682926238708,-0.12160616508988303,0.055364844397799096,-1.0494471430472119,0 +2.0797923367508915,2.219685381971847,1.0991206914633669,0.6908645211994979,-0.2687187202798915,0 +-1.6068619653777816,0.8764512389950713,2.3960177761739483,1.6928703374643554,-2.2920675623752054,0 +-2.1594585035736724,-1.6965939809476205,-1.1597192926874123,-0.0753772628669711,1.6385657377017577,1 +0.12077300472544472,-2.2483790492672533,0.8752849092713,1.9238991697722891,-2.2118280265977344,0 +0.34950713997137406,-2.2806240662815633,-0.3839319376785896,1.777298999423122,0.44702007646501807,1 +-0.2654553193635607,-0.46665847180021536,1.1346525343627611,1.3545976890463567,-1.5265641360643252,0 +-1.856611656158378,1.8524465492466156,1.949048149426567,0.932439892243901,-1.0275649872080315,1 +-1.2016280711053537,-0.9902722472058406,-1.0356992565732115,-0.6252810457737985,1.131203827680352,1 +-1.368043876382231,1.1789889846120305,-0.5100459367728748,-0.21309019000612217,2.5926008437243695,1 +0.8942693852357111,2.139071310601736,1.8799228844904594,1.4138011695114332,-0.9721173643339782,0 +-2.4060024939544356,0.9508899388747825,1.9507171769888003,2.7220330455466373,0.219885441850727,1 +-1.1331863863469285,-2.9626780781585627,-0.8011123459938203,1.9250101376335575,1.3938449167471199,1 +-2.647914324599812,-1.3610202151019353,-1.3348239104925301,-0.4235076276249292,2.162772596237362,1 +-0.055140663699368764,1.2881848437048036,1.5290617851992776,1.5559137520965856,-0.5338645841188343,0 +-1.6546110439720927,-0.025686262282198058,-0.017499025571771964,0.6497472140291349,1.2855194211162537,1 +2.975574761230523,2.419820582192545,0.6785516902215438,0.03850874460031484,-0.1086097527953227,0 +1.0763513351130107,2.1294094611406873,1.2079760885374984,0.8249951886323986,-0.12813505345106035,0 +-1.591954595850689,-0.31830491433761376,-0.20804561224971763,0.47376610897359195,1.2206523729895955,1 +-0.5663335747054519,-0.7901233485646082,0.19897067861984086,1.636826317019823,0.7376159657800063,1 +-0.6472819629840465,-1.3938145754481424,-1.078652842488317,-0.8949624948329402,0.3436757090900173,1 +2.3964289711656575,1.3840583004891485,0.05674820503788647,-0.36640690353630334,0.04800716939780858,0 +-1.5879749726823809,1.4097794209321337,1.9999909308327104,1.571841285517877,-0.97578654146721,0 +-2.0608784170949264,-1.9292752321736781,-2.600790564303219,-1.276013658868342,3.4024582518300086,1 +0.6360578412575373,2.651867689418183,-0.15325948626672936,-0.7106250954850648,2.01885855535229,1 +1.1063828414340589,0.8691778993427344,0.6049688452042938,0.2859114573080176,-0.5923514635433662,0 +-1.0584031106409864,1.892100113913194,1.9215988006883913,1.3792377301184393,-0.6996833151518032,0 +-3.322033044786915,-1.4002695991049532,-1.6410013316373464,-1.2604556973060097,2.140143987446071,1 +-1.8763315768478992,0.2764298476750694,0.948597969426878,2.1015798196408677,1.0141053190223035,1 +-1.0291014407041037,-0.02065552711684937,3.8827450770240564,4.385191955998962,-3.8788530177379528,0 +0.6875135596368191,-2.5821662620414925,-1.3972523899406486,0.08643189645327709,0.5266243319040311,1 +0.7128506438784368,5.737793131621844,2.270182182523157,0.8842600520778051,1.205114885288078,1 +-2.552191918547829,0.9978962939519059,1.0137223518719514,1.5274720692791357,1.180294132547354,1 +-0.25950429659657837,2.606343069201997,0.4811422093989864,-0.7456597418429676,0.7592336830283972,0 +0.6119053248109316,1.032538881515911,0.22549898505907245,-0.03447961375704134,0.26721393410869365,0 +-1.3343520113214837,0.6556699815298489,1.7419807984145028,1.3514252678511829,-1.4593073086303021,0 +-1.8332962405846545,1.2603644391209619,2.29984563806238,1.0962541438293516,-2.264563429080373,0 +-0.04551418261958373,1.1359760557676202,1.6828664813100631,1.4547604010169624,-1.1588624216666135,0 +-1.5141736970725765,-1.487787355228758,-0.9555348395511414,0.5311916604468465,1.8243978856794336,1 +2.7974314238344116,0.9879068057949466,-0.575225216168288,-1.0997597619749349,0.17896071281217574,0 +0.5607124807115611,-0.4499914267628853,0.45555425739311306,1.8494412145170505,0.33775024286845745,1 +-0.8227790111401876,-0.1419745159858632,1.797990801211173,2.062431188491476,-1.7777878659828437,0 +-1.3021306380180802,2.7445906607932438,1.7914988948026433,1.8207365854764395,1.0254696023941723,1 +-1.6751532434457437,-1.0594884596578384,0.2019177616490525,1.9402622556553806,1.1668499188225656,1 +0.5090246062619943,1.2462977627755303,1.888570960302307,1.6835622311737195,-1.4608607071889814,0 +-2.9213691325281457,-1.2738835801340296,-0.6406463428319629,0.528208217266391,1.7750375295167942,1 +-0.8882140219876686,0.8607145646432016,1.6762960658564352,1.311419279698875,-1.2946758951498674,0 +-0.9483225547090204,-0.5662366331511879,1.6018056835444354,1.9950467368749156,-1.7787768515802276,0 +-0.7318007391528466,2.0407903746494496,1.1044514574069897,0.4893977241783146,0.2608261345698315,0