+ * This is required, as default {@link WebSocketServletFactory} uses {@link org.eclipse.jetty.util.DecoratedObjectFactory}
+ * to instantiate {@link WebSocketListener} classes. This class is only able to instantiate static classes with 0-arg constructor,
+ * which inner non-static class {@link Jetty9WebsocketHandler} is NOT.
+ */
+ public class H2OWebSocketCreator implements WebSocketCreator {
+
+ @Override
+ public Object createWebSocket(ServletUpgradeRequest req, ServletUpgradeResponse resp) {
+ return new Jetty9WebsocketHandler();
+ }
+ }
+}
diff --git a/h2o-parsers/h2o-avro-parser/build.gradle b/h2o-parsers/h2o-avro-parser/build.gradle
index be498665d8ed..d61f58043d90 100644
--- a/h2o-parsers/h2o-avro-parser/build.gradle
+++ b/h2o-parsers/h2o-avro-parser/build.gradle
@@ -6,7 +6,7 @@ description = "H2O Avro Parser"
dependencies {
api project(":h2o-core")
// Avro support
- api 'org.apache.avro:avro:1.11.3'
+ api 'org.apache.avro:avro:1.11.4'
testImplementation project(":h2o-test-support")
testRuntimeOnly project(":${defaultWebserverModule}")
diff --git a/h2o-parsers/h2o-parquet-parser/src/main/java/water/parser/parquet/FrameParquetExporter.java b/h2o-parsers/h2o-parquet-parser/src/main/java/water/parser/parquet/FrameParquetExporter.java
index 8c35cc72c8b8..e1fc1f8a85ed 100644
--- a/h2o-parsers/h2o-parquet-parser/src/main/java/water/parser/parquet/FrameParquetExporter.java
+++ b/h2o-parsers/h2o-parquet-parser/src/main/java/water/parser/parquet/FrameParquetExporter.java
@@ -112,7 +112,9 @@ public void map(Chunk[] cs) {
group = group.append(currColName, cs[j].at8(i));
break;
case (T_STR):
- group = group.append(currColName, cs[j].atStr(new BufferedString(), i).toString());
+ if (!cs[j].isNA(i)) {
+ group = group.append(currColName, cs[j].atStr(new BufferedString(), i).toString());
+ }
break;
case (T_CAT):
if (cs[j].isNA(i)) {
diff --git a/h2o-persist-gcs/build.gradle b/h2o-persist-gcs/build.gradle
index 5d1ba374ad2b..d07448510c39 100644
--- a/h2o-persist-gcs/build.gradle
+++ b/h2o-persist-gcs/build.gradle
@@ -4,10 +4,18 @@ description = "H2O Persist GCS"
dependencies {
api project(":h2o-core")
- api 'com.google.cloud:google-cloud-storage:2.13.1'
+ api ('com.google.cloud:google-cloud-storage:2.13.1')
testImplementation project(":h2o-test-support")
testRuntimeOnly project(":${defaultWebserverModule}")
+
+ constraints {
+ api('com.google.protobuf:protobuf-java:3.25.5') {
+ because 'Fixes CVE-2024-7254'
+ because 'Fixes SNYK-JAVA-COMGOOGLEPROTOBUF-8055227'
+ because 'Fixes SNYK-JAVA-COMGOOGLEPROTOBUF-8055228'
+ }
+ }
}
apply from: "${rootDir}/gradle/dataCheck.gradle"
diff --git a/h2o-py/h2o/display.py b/h2o-py/h2o/display.py
index 6815c19a1f79..5b382433605c 100644
--- a/h2o-py/h2o/display.py
+++ b/h2o-py/h2o/display.py
@@ -5,6 +5,7 @@
:copyright: (c) 2016 H2O.ai
:license: Apache License Version 2.0 (see LICENSE for details)
"""
+# when changing this module, please make sure it doesn't break explanations in jupyter, vscode and ipython
from contextlib import contextmanager
import os
import sys
diff --git a/h2o-py/h2o/estimators/decision_tree.py b/h2o-py/h2o/estimators/decision_tree.py
index e598396b2a82..0e7f391b515f 100644
--- a/h2o-py/h2o/estimators/decision_tree.py
+++ b/h2o-py/h2o/estimators/decision_tree.py
@@ -107,6 +107,22 @@ def ignore_const_cols(self):
Ignore constant columns.
Type: ``bool``, defaults to ``True``.
+
+ :examples:
+
+ >>> import h2o
+ >>> from h2o.estimators import H2ODecisionTreeEstimator
+ >>> h2o.init()
+ >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv")
+ >>> target_variable = 'CAPSULE'
+ >>> prostate[target_variable] = prostate[target_variable].asfactor()
+ >>> prostate["const_1"] = 6
+ >>> train, test = prostate.split_frame(ratios=[0.7])
+ >>> sdt_h2o = H2ODecisionTreeEstimator(model_id="decision_tree.hex",
+ ... max_depth=5,
+ ... ignore_const_cols=True)
+ >>> sdt_h2o.train(y=target_variable, training_frame=train)
+ >>> pred_test = sdt_h2o.predict(test)
"""
return self._parms.get("ignore_const_cols")
@@ -122,6 +138,22 @@ def categorical_encoding(self):
Type: ``Literal["auto", "enum", "one_hot_internal", "one_hot_explicit", "binary", "eigen", "label_encoder",
"sort_by_response", "enum_limited"]``, defaults to ``"auto"``.
+
+ :examples:
+
+ >>> import h2o
+ >>> from h2o.estimators import H2ODecisionTreeEstimator
+ >>> h2o.init()
+ >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv")
+ >>> target_variable = 'CAPSULE'
+ >>> prostate["RACE"] = prostate["RACE"].asfactor()
+ >>> prostate[target_variable] = prostate[target_variable].asfactor()
+ >>> train, test = prostate.split_frame(ratios=[0.7])
+ >>> sdt_h2o = H2ODecisionTreeEstimator(model_id="decision_tree.hex",
+ ... max_depth=5,
+ ... categorical_encoding="binary")
+ >>> sdt_h2o.train(y=target_variable, training_frame=train)
+ >>> pred_test = sdt_h2o.predict(test)
"""
return self._parms.get("categorical_encoding")
@@ -164,6 +196,20 @@ def max_depth(self):
Max depth of tree.
Type: ``int``, defaults to ``20``.
+
+ :examples:
+
+ >>> import h2o
+ >>> from h2o.estimators import H2ODecisionTreeEstimator
+ >>> h2o.init()
+ >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv")
+ >>> target_variable = 'CAPSULE'
+ >>> prostate[target_variable] = prostate[target_variable].asfactor()
+ >>> train, test = prostate.split_frame(ratios=[0.7])
+ >>> sdt_h2o = H2ODecisionTreeEstimator(model_id="decision_tree.hex",
+ ... max_depth=5)
+ >>> sdt_h2o.train(y=target_variable, training_frame=train)
+ >>> pred_test = sdt_h2o.predict(test)
"""
return self._parms.get("max_depth")
@@ -178,6 +224,21 @@ def min_rows(self):
Fewest allowed (weighted) observations in a leaf.
Type: ``int``, defaults to ``10``.
+
+ :examples:
+
+ >>> import h2o
+ >>> from h2o.estimators import H2ODecisionTreeEstimator
+ >>> h2o.init()
+ >>> prostate = h2o.import_file("http://s3.amazonaws.com/h2o-public-test-data/smalldata/prostate/prostate.csv")
+ >>> target_variable = 'CAPSULE'
+ >>> prostate[target_variable] = prostate[target_variable].asfactor()
+ >>> train, test = prostate.split_frame(ratios=[0.7])
+ >>> sdt_h2o = H2ODecisionTreeEstimator(model_id="decision_tree.hex",
+ ... max_depth=5,
+ ... min_rows=20)
+ >>> sdt_h2o.train(y=target_variable, training_frame=train)
+ >>> pred_test = sdt_h2o.predict(test)
"""
return self._parms.get("min_rows")
diff --git a/h2o-py/h2o/estimators/rulefit.py b/h2o-py/h2o/estimators/rulefit.py
index 529b371780ea..be80309794b0 100644
--- a/h2o-py/h2o/estimators/rulefit.py
+++ b/h2o-py/h2o/estimators/rulefit.py
@@ -206,6 +206,22 @@ def algorithm(self):
The algorithm to use to generate rules.
Type: ``Literal["auto", "drf", "gbm"]``, defaults to ``"auto"``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... algorithm="gbm",
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("algorithm")
@@ -220,6 +236,22 @@ def min_rule_length(self):
Minimum length of rules. Defaults to 3.
Type: ``int``, defaults to ``3``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... min_rule_length=4,
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("min_rule_length")
@@ -234,6 +266,22 @@ def max_rule_length(self):
Maximum length of rules. Defaults to 3.
Type: ``int``, defaults to ``3``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... min_rule_length=3,
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("max_rule_length")
@@ -249,6 +297,21 @@ def max_num_rules(self):
by diminishing returns in model deviance.
Type: ``int``, defaults to ``-1``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=3,
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("max_num_rules")
@@ -263,6 +326,22 @@ def model_type(self):
Specifies type of base learners in the ensemble.
Type: ``Literal["rules_and_linear", "rules", "linear"]``, defaults to ``"rules_and_linear"``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... model_type="rules",
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("model_type")
@@ -298,6 +377,22 @@ def distribution(self):
Type: ``Literal["auto", "bernoulli", "multinomial", "gaussian", "poisson", "gamma", "tweedie", "laplace",
"quantile", "huber"]``, defaults to ``"auto"``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... distribution="bernoulli",
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("distribution")
@@ -312,6 +407,22 @@ def rule_generation_ntrees(self):
Specifies the number of trees to build in the tree model. Defaults to 50.
Type: ``int``, defaults to ``50``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... rule_generation_ntrees=60,
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("rule_generation_ntrees")
@@ -370,6 +481,22 @@ def max_categorical_levels(self):
for categorical_encoding == EnumLimited.
Type: ``int``, defaults to ``10``.
+
+ :examples:
+
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... max_categorical_levels=11,
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> print(rfit.rule_importance())
"""
return self._parms.get("max_categorical_levels")
@@ -385,6 +512,21 @@ def rule_importance(self):
Retrieve rule importances for a Rulefit model
:return: H2OTwoDimTable
+
+ :examples:
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/gbm_test/titanic.csv"
+ >>> df = h2o.import_file(path=f, col_types={'pclass': "enum", 'survived': "enum"})
+ >>> x = ["age", "sibsp", "parch", "fare", "sex", "pclass"]
+ >>> y = "survived"
+ >>> rfit = H2ORuleFitEstimator(max_rule_length=10,
+ ... max_num_rules=100,
+ ... seed=1)
+ >>> rfit.train(training_frame=df, x=x, y=y)
+ >>> rule_importance = rfit.rule_importance()
+ >>> print(rfit.rule_importance())
"""
if self._model_json["algo"] != "rulefit":
raise H2OValueError("This function is available for Rulefit models only")
@@ -397,11 +539,29 @@ def rule_importance(self):
def predict_rules(self, frame, rule_ids):
"""
- Evaluates validity of the given rules on the given data.
+ Evaluates validity of the given rules on the given data.
:param frame: H2OFrame on which rule validity is to be evaluated
:param rule_ids: string array of rule ids to be evaluated against the frame
:return: H2OFrame with a column per each input ruleId, representing a flag whether given rule is applied to the observation or not.
+
+ :examples:
+ >>> import h2o
+ >>> h2o.init()
+ >>> from h2o.estimators import H2ORuleFitEstimator
+ >>> f = "https://s3.amazonaws.com/h2o-public-test-data/smalldata/iris/iris_train.csv"
+ >>> df = h2o.import_file(path=f, col_types={'species': "enum"})
+ >>> x = df.columns
+ >>> y = "species"
+ >>> x.remove(y)
+ >>> train, test = df.split_frame(ratios=[.8], seed=1234)
+ >>> rfit = H2ORuleFitEstimator(min_rule_length=4,
+ ... max_rule_length=5,
+ ... max_num_rules=3,
+ ... seed=1234,
+ ... model_type="rules")
+ >>> rfit.train(training_frame=train, x=x, y=y, validation_frame=test)
+ >>> print(rfit.predict_rules(train, ['M0T38N5_Iris-virginica']))
"""
from h2o.frame import H2OFrame
from h2o.utils.typechecks import assert_is_type
diff --git a/h2o-py/h2o/frame.py b/h2o-py/h2o/frame.py
index 42726858deba..432e197016cb 100644
--- a/h2o-py/h2o/frame.py
+++ b/h2o-py/h2o/frame.py
@@ -28,7 +28,7 @@
from h2o.utils.metaclass import deprecated_fn
from h2o.utils.shared_utils import(gen_header, is_list, is_list_of_lists, is_str_list, py_tmp_key, quoted,
can_use_pandas, can_use_numpy, quote, normalize_slice, slice_is_normalized,
- check_frame_id, can_use_datatable, can_use_polars, can_use_pyarrow)
+ check_frame_id, can_use_polars, can_use_pyarrow)
from h2o.utils.threading import local_context, local_env
from h2o.utils.typechecks import (assert_is_type, assert_satisfies, Enum, I, is_type, numeric, numpy_ndarray,
numpy_datetime, pandas_dataframe, pandas_timestamp, scipy_sparse, U)
@@ -1942,17 +1942,16 @@ def structure(self):
else:
print("num {}".format(" ".join(it[0] if it else "nan" for it in h2o.as_list(self[:10, i], False)[1:])))
- def as_data_frame(self, use_pandas=True, header=True):
+ def as_data_frame(self, use_pandas=True, header=True, use_multi_thread=False):
"""
Obtain the dataset as a python-local object.
:param bool use_pandas: If True (default) then return the H2OFrame as a pandas DataFrame (requires that the
``pandas`` library was installed). If False, then return the contents of the H2OFrame as plain nested
- list, in a row-wise order. The conversion to pandas frame will use multi-thread whenever
- possible with the right python modules (datatable or polars and pyarrow) installed. Otherwise, single
- thread operation will be used in the conversion.
+ list, in a row-wise order.
:param bool header: If True (default), then column names will be appended as the first row in list
-
+ :param bool use_multi_thread: If True (False by default), will use polars/pyarrow to perform conversion in
+ multi-thread which is faster.
:returns: A python object (a list of lists of strings, each list is a row, if ``use_pandas=False``, otherwise
a pandas DataFrame) containing this H2OFrame instance's data.
@@ -1969,22 +1968,19 @@ def as_data_frame(self, use_pandas=True, header=True):
"""
if can_use_pandas() and use_pandas:
import pandas
- if (can_use_datatable()) or (can_use_polars() and can_use_pyarrow()): # can use multi-thread
- exportFile = tempfile.NamedTemporaryFile(suffix=".h2oframe2Convert.csv", delete=False)
- try:
- exportFile.close() # needed for Windows
- h2o.export_file(self, exportFile.name, force=True)
- if can_use_datatable(): # use datatable for multi-thread by default
- return self.convert_with_datatable(exportFile.name)
- elif can_use_polars() and can_use_pyarrow(): # polar/pyarrow if datatable is not available
- return self.convert_with_polars(exportFile.name)
- finally:
- os.unlink(exportFile.name)
+ if use_multi_thread:
+ with local_context(polars_enabled=True): # turn on multi-thread toolboxes
+ if can_use_polars() and can_use_pyarrow(): # can use multi-thread
+ exportFile = tempfile.NamedTemporaryFile(suffix=".h2oframe2Convert.csv", delete=False)
+ try:
+ exportFile.close() # needed for Windows
+ h2o.export_file(self, exportFile.name, force=True)
+ return self.convert_with_polars(exportFile.name)
+ finally:
+ os.unlink(exportFile.name)
warnings.warn("Converting H2O frame to pandas dataframe using single-thread. For faster conversion using"
- " multi-thread, install datatable (for Python 3.9 or lower), or polars and pyarrow "
- "(for Python 3.10 or above) and activate it using:\n\n"+
- "with h2o.utils.threading.local_context(polars_enabled=True, datatable_enabled=True):\n"
- " pandas_df = h2o_df.as_data_frame()\n", H2ODependencyWarning)
+ " multi-thread, install polars and pyarrow and use it as "
+ "pandas_df = h2o_df.as_data_frame(use_multi_thread=True)\n", H2ODependencyWarning)
return pandas.read_csv(StringIO(self.get_frame_data()), low_memory=False, skip_blank_lines=False)
from h2o.utils.csv.readers import reader
@@ -1998,18 +1994,6 @@ def convert_with_polars(self, fileName):
import polars as pl
dt_frame = pl.read_csv(fileName, null_values = "")
return dt_frame.to_pandas()
-
- def convert_with_datatable(self, fileName):
- import datatable as dt
- frameTypes = self.types
- validFrameTypes = {}
- for key, value in frameTypes.items():
- if value.startswith('int'):
- validFrameTypes[key] = dt.int64
- elif value.startswith("real"):
- validFrameTypes[key] = dt.float64
- dt_frame = dt.fread(fileName, na_strings=[""], columns=validFrameTypes)
- return dt_frame.to_pandas()
def save_to_hive(self, jdbc_url, table_name, format="csv", table_path=None, tmp_path=None):
"""
diff --git a/h2o-py/h2o/h2o.py b/h2o-py/h2o/h2o.py
index 542cb117d872..db67d4fb93be 100644
--- a/h2o-py/h2o/h2o.py
+++ b/h2o-py/h2o/h2o.py
@@ -868,14 +868,22 @@ def parse_setup(raw_frames, destination_frame=None, header=0, separator=None, co
if ind in skipped_columns:
use_type[ind]=False
- if column_names is not None:
+ if column_names is not None:
if not isinstance(column_names, list): raise ValueError("col_names should be a list")
if (skipped_columns is not None) and len(skipped_columns)>0:
- if (len(column_names)) != parse_column_len:
+ # when we are converting a python object to H2OFrame, column_names will include all columns despite
+ # skipped columns are specified. In this case, we need to make sure that
+ # len(column_names)-len(skipped_columns)==parse_column_len
+ # When we are importing a file with skipped columns mentioned, column_names will only contain columns that
+ # are not skipped. Hence, in this case, we need to check len(column_names) == parse_column_len.
+ # To combine the two, correct parsing will have conditions len(column_names)-len(skipped_columns)==parse_column_len
+ # or len(column_names)==parse_column_len. Hence, we will raise an error when
+ # not(len(column_names)-len(skipped_columns)==parse_column_len or len(column_names)==parse_column_len happened.
+ if not((len(column_names) == parse_column_len) or ((len(column_names)-len(skipped_columns))==parse_column_len)):
raise ValueError(
- "length of col_names should be equal to the number of columns parsed: %d vs %d"
- % (len(column_names), parse_column_len))
- else:
+ "length of col_names minus length of skipped_columns should equal the number of columns parsed: "
+ "%d vs %d" % (len(column_names), parse_column_len))
+ else: # no skipped columns here
if len(column_names) != len(j["column_types"]): raise ValueError(
"length of col_names should be equal to the number of columns: %d vs %d"
% (len(column_names), len(j["column_types"])))
diff --git a/h2o-py/h2o/plot/_matplotlib.py b/h2o-py/h2o/plot/_matplotlib.py
index fa0b4212041f..d7e97caf541c 100644
--- a/h2o-py/h2o/plot/_matplotlib.py
+++ b/h2o-py/h2o/plot/_matplotlib.py
@@ -1,9 +1,11 @@
def get_matplotlib_pyplot(server, raise_if_not_available=False):
+ # when changing this function, please make sure it doesn't break explanations in jupyter, vscode and ipython
try:
# noinspection PyUnresolvedReferences
import matplotlib
- matplotlib.use("Agg")
+ if server:
+ matplotlib.use("Agg")
try:
# noinspection PyUnresolvedReferences
import matplotlib.pyplot as plt
@@ -25,6 +27,7 @@ def get_matplotlib_pyplot(server, raise_if_not_available=False):
def get_polycollection(server, raise_if_not_available=False):
+ # when changing this function, please make sure it doesn't break explanations in jupyter, vscode and ipython
try:
from matplotlib.collections import PolyCollection as polycoll
return polycoll
@@ -36,6 +39,7 @@ def get_polycollection(server, raise_if_not_available=False):
def get_matplotlib_cm(function_name):
+ # when changing this function, please make sure it doesn't break explanations in jupyter, vscode and ipython
try:
from matplotlib import cm
return cm
@@ -45,6 +49,7 @@ def get_matplotlib_cm(function_name):
def get_mplot3d_axes(function_name):
+ # when changing this function, please make sure it doesn't break explanations in jupyter, vscode and ipython
try:
# noinspection PyUnresolvedReferences
from mpl_toolkits.mplot3d import Axes3D
diff --git a/h2o-py/h2o/plot/_plot_result.py b/h2o-py/h2o/plot/_plot_result.py
index ad8b3bab2d87..25e6642957af 100644
--- a/h2o-py/h2o/plot/_plot_result.py
+++ b/h2o-py/h2o/plot/_plot_result.py
@@ -1,5 +1,6 @@
# -*- encoding: utf-8 -*-
# mutable versions of py immutable types
+# when changing this module, please make sure it doesn't break explanations in jupyter, vscode and ipython
from h2o.exceptions import H2OError
__no_export = set(dir()) # all variables defined above this are not exported
diff --git a/h2o-py/h2o/utils/shared_utils.py b/h2o-py/h2o/utils/shared_utils.py
index 67b622fd62c6..89356c1144ed 100644
--- a/h2o-py/h2o/utils/shared_utils.py
+++ b/h2o-py/h2o/utils/shared_utils.py
@@ -137,28 +137,19 @@ def is_module_enabled(mod):
def can_use_pandas():
return is_module_available('pandas')
-
-def can_use_datatable():
- return is_module_enabled('datatable') and sys.version_info.major == 3 and sys.version_info.minor <= 9
-
-
-def can_install_datatable():
- return sys.version_info.major == 3 and sys.version_info.minor <= 9
-
-
def can_install_polars():
- return sys.version_info.major == 3 and sys.version_info.minor > 9
+ return sys.version_info.major == 3 and sys.version_info.minor >= 6
def can_use_polars():
- return is_module_enabled('polars') and sys.version_info.major == 3 and sys.version_info.minor > 9
+ return is_module_enabled('polars') and sys.version_info.major == 3 and sys.version_info.minor >= 6
def can_use_pyarrow():
- if can_use_pandas() and sys.version_info.minor > 9:
+ if can_use_pandas() and sys.version_info.minor >= 6:
import pandas
- return is_module_available('pyarrow') and sys.version_info.major == 3 and sys.version_info.minor > 9 and \
- sys.version_info.major == 3 and float(pandas.__version__[0]) >= 1
+ return is_module_available('pyarrow') and sys.version_info.major == 3 and sys.version_info.minor >= 6 and \
+ float(pandas.__version__[0]) >= 1
else:
return False
diff --git a/h2o-py/tests/pyunit_utils/__init__.py b/h2o-py/tests/pyunit_utils/__init__.py
index b2987ec0859d..a8415192911b 100644
--- a/h2o-py/tests/pyunit_utils/__init__.py
+++ b/h2o-py/tests/pyunit_utils/__init__.py
@@ -3,3 +3,4 @@
from .utils_model_custom_distribution import *
from .utils_for_glm_tests import *
from .sklearn_multinomial_auc_method import roc_auc_score
+from .utils_parser_tests import *
diff --git a/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py b/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py
index 010845eb56e0..f993375e9578 100644
--- a/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py
+++ b/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py
@@ -1,4 +1,3 @@
-import h2o
from h2o.estimators import H2OGeneralizedLinearEstimator as glm
from h2o.exceptions import H2OValueError
from h2o.grid.grid_search import H2OGridSearch
diff --git a/h2o-py/tests/pyunit_utils/utils_parser_tests.py b/h2o-py/tests/pyunit_utils/utils_parser_tests.py
new file mode 100644
index 000000000000..4993887032eb
--- /dev/null
+++ b/h2o-py/tests/pyunit_utils/utils_parser_tests.py
@@ -0,0 +1,45 @@
+from tests import pyunit_utils
+import h2o
+import time
+import pandas as pd
+
+def test_frame_conversion(dataset, original_pandas_frame):
+ # convert frame using datatable or polar/pyarrow
+ h2oframe = h2o.import_file(pyunit_utils.locate(dataset))
+ test_frames_conversion(h2oframe, original_pandas_frame)
+
+def test_frames_conversion(h2oframe, original_pandas_frame):
+ start_time = time.time()
+ new_pandas_frame = h2oframe.as_data_frame(use_multi_thread=True)
+ new_time = time.time()-start_time
+ print("H2O frame to Pandas frame conversion time with multi-thread using module polars/pyarrow: {0}".format(new_time))
+ # compare two frames column types
+ new_types = new_pandas_frame.dtypes
+ old_types = original_pandas_frame.dtypes
+ ncol = h2oframe.ncol
+ col_names = new_pandas_frame.columns
+
+ for ind in list(range(ncol)):
+ assert new_types[col_names[ind]] == old_types[col_names[ind]], "Expected column types: {0}, actual column types: " \
+ "{1}".format(old_types[col_names[ind]], new_types[col_names[ind]])
+ if new_types[col_names[ind]] == "object":
+ diff = new_pandas_frame[col_names[ind]] == original_pandas_frame[col_names[ind]]
+ if not diff.all(): # difference caused by the presence of NAs
+ new_series = pd.Series(new_pandas_frame[col_names[ind]])
+ new_NA = new_series.isna()
+ old_series = pd.Series(original_pandas_frame[col_names[ind]])
+ old_NA = old_series.isna()
+ assert (new_NA==old_NA).all()
+ else:
+ diff = (new_pandas_frame[col_names[ind]] - original_pandas_frame[col_names[ind]]).abs()
+ assert diff.max() < 1e-10
+
+
+def single_thread_pandas_conversion(dataset):
+ print("converting h2o frame to pandas frame using single thread")
+ h2oframe = h2o.import_file(pyunit_utils.locate(dataset))
+ start_time = time.time()
+ h2oframe_panda = h2oframe.as_data_frame()
+ new_time = time.time()-start_time
+ print("H2O frame to Pandas frame conversion time with single thread for dataset {1}: {0}".format(new_time, dataset))
+ return h2oframe_panda
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_bad_constraints_large.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_bad_constraints_large.py
new file mode 100644
index 000000000000..c9a81a79a75f
--- /dev/null
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_bad_constraints_large.py
@@ -0,0 +1,156 @@
+import h2o
+from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm
+import numpy as np
+import pandas as pd
+from tests import pyunit_utils
+
+# this test needs to run into completion duplicating/conflicting constraints
+def data_prep(seed):
+ np.random.seed(seed)
+ x1 = np.random.normal(0, 10, 100000)
+ x2 = np.random.normal(10, 100 , 100000)
+ x3 = np.random.normal(20, 200, 100000)
+ x4 = np.random.normal(30, 3000, 100000)
+ x5 = np.random.normal(400, 4000, 100000)
+
+ y_raw = np.sin(x1)*100 + np.sin(x2)*100 + x3/20 + x3/30 + x5/400
+ y = np.random.normal(y_raw, 20)
+
+ data = {
+ 'x1': x1,
+ 'x2': x2,
+ 'x3': x3,
+ 'x4': x4,
+ 'x5': x5,
+ 'y': y,
+ }
+ return h2o.H2OFrame(pd.DataFrame(data))
+
+def test_duplicate_conflicting_constraints():
+ train_data = data_prep(123)
+ family = 'gaussian'
+ link = 'identity'
+ nfolds = 0
+ lambda_ = 0.0
+ seed = 1234
+ calc_like = True
+ compute_p_values = True
+ solver = 'irlsm'
+ predictors = ['x1', 'x2', 'x3', 'x4', 'x5']
+ response = "y"
+
+ linear_constraints2 = []
+
+ name = "x2"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = -1
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x4"
+ values = -1
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x2"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x4"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ linear_constraints = h2o.H2OFrame(linear_constraints2)
+ linear_constraints.set_names(["names", "values", "types", "constraint_numbers"])
+
+ params = {
+ "family" : family,
+ "link": link,
+ "lambda_" : lambda_,
+ "seed" : seed,
+ "nfolds" : nfolds,
+ "compute_p_values" : compute_p_values,
+ "calc_like" : calc_like,
+ "solver" : solver,
+ "linear_constraints": linear_constraints
+ }
+
+ model = glm(**params)
+ model.train(x = predictors, y = response, training_frame = train_data)
+ print(model.coef())
+ coef_constrained = model.coef()
+ print(glm.getConstraintsInfo(model))
+
+ params = {
+ "family" : family,
+ "link": link,
+ "lambda_" : lambda_,
+ "seed" : seed,
+ "nfolds" : nfolds,
+ "compute_p_values" : compute_p_values,
+ "calc_like" : calc_like,
+ "solver" : solver,
+ }
+
+ model_no_constraints = glm(**params)
+ model_no_constraints.train(x = predictors, y = response, training_frame = train_data)
+ coef_no_constraints = model_no_constraints.coef()
+ print("model built without constraints")
+ print(coef_no_constraints)
+ print("x2-x3: {0}".format(coef_no_constraints['x2']-coef_no_constraints['x3']))
+ print("x3-x4: {0}".format(coef_no_constraints['x3']-coef_no_constraints['x4']))
+ print("x2+x3+x4: {0}".format(coef_no_constraints['x2']+coef_no_constraints['x3']+coef_no_constraints['x4']))
+ # assert that model with linear constraints does a better job than model without constraints
+ assert (coef_constrained['x2']-coef_constrained['x3']) < (coef_no_constraints['x2']-coef_no_constraints['x3']), \
+ "Model built with constraints should be closer to the constraint x2-x3 <= 0"
+ assert (coef_constrained['x3']-coef_constrained['x4']) < (coef_no_constraints['x3']-coef_no_constraints['x4']), \
+ "Model built with constraints should be closer to the constraint x3-x4 <= 0"
+ assert (coef_constrained['x2']+coef_constrained['x3']+coef_constrained['x4']) < \
+ (coef_no_constraints['x2']+coef_no_constraints['x3']+coef_no_constraints['x4']), \
+ "Model built with constraints should be closer to the constraint x2+x3+x4 <= 0"
+
+if __name__ == "__main__":
+ pyunit_utils.standalone_test(test_duplicate_conflicting_constraints)
+else:
+ test_duplicate_conflicting_constraints()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_beta_constraint_NPE_large.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_beta_constraint_NPE_large.py
new file mode 100644
index 000000000000..7c7b18ef72c7
--- /dev/null
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_beta_constraint_NPE_large.py
@@ -0,0 +1,163 @@
+import h2o
+from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm
+from tests import pyunit_utils
+import numpy as np
+import pandas as pd
+
+# For beta constraints, if only upper_bounds are specified, there are NPE errors because the code expects both upper
+# and lower bounds to be specified. I have since fixed this error.
+def data_prep(seed):
+ np.random.seed(seed)
+ x1 = np.random.normal(0, 10, 100000)
+ x2 = np.random.normal(10, 100 , 100000)
+ x3 = np.random.normal(20, 200, 100000)
+ x4 = np.random.normal(30, 3000, 100000)
+ x5 = np.random.normal(400, 4000, 100000)
+
+ y_raw = np.sin(x1)*100 + np.sin(x2)*100 + x3/20 + x3/30 + x5/400
+ y = np.random.normal(y_raw, 20)
+
+ data = {
+ 'x1': x1,
+ 'x2': x2,
+ 'x3': x3,
+ 'x4': x4,
+ 'x5': x5,
+ 'y': y,
+ }
+ return h2o.H2OFrame(pd.DataFrame(data))
+
+def test_bad_lambda_specification():
+ train_data = data_prep(123)
+ family = 'gaussian'
+ link = 'identity'
+ nfolds = 0
+ lambda_ = 0.0
+ seed = 1234
+ calc_like = True
+ compute_p_values = True
+ solver = 'irlsm'
+ predictors = ['x1', 'x2', 'x3', 'x4', 'x5']
+ response = "y"
+
+ # beta constraints
+ bc = []
+ name = 'x1'
+ lower_bound = 0.03
+ bc.append([name, lower_bound])
+
+ beta_constraints = h2o.H2OFrame(bc)
+ beta_constraints.set_names(["names", "lower_bounds"])
+
+ linear_constraints2 = []
+
+ name = "x2"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = -1
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x4"
+ values = -1
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x2"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x4"
+ values = 1
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+
+ linear_constraints = h2o.H2OFrame(linear_constraints2)
+ linear_constraints.set_names(["names", "values", "types", "constraint_numbers"])
+
+ linear_constraints = h2o.H2OFrame(linear_constraints2)
+ linear_constraints.set_names(["names", "values", "types", "constraint_numbers"])
+ # check lower bound of beta constraint will not generate error but lambda will.
+ params = {
+ "family" : family,
+ "link": link,
+ "lambda_" : lambda_,
+ "seed" : seed,
+ "nfolds" : nfolds,
+ "compute_p_values" : compute_p_values,
+ "calc_like" : calc_like,
+ "solver" : solver,
+ "linear_constraints": linear_constraints,
+ "beta_constraints": beta_constraints
+ }
+
+ model = glm(**params)
+ model.train(x = predictors, y = response, training_frame = train_data)
+ coefs = model.coef()
+ print(coefs)
+ print(glm.getConstraintsInfo(model))
+ # beta constraints should be satisfied
+ assert coefs["x1"] >= 0.03 or abs(coefs["x1"]-0.03) < 1e-6, "beta constraint x1 ({0}) >= 0.03 is violated!".format(coefs["x1"])
+
+ # beta constraints
+ bc = []
+ name = 'x1'
+ upper_bound = 1.5
+ bc.append([name, upper_bound])
+
+ beta_constraints2 = h2o.H2OFrame(bc)
+ beta_constraints2.set_names(["names", "upper_bounds"])
+
+ params['beta_constraints'] = beta_constraints2
+ model = glm(**params)
+ model.train(x = predictors, y = response, training_frame = train_data)
+ coefs = model.coef()
+ print(coefs)
+ print(glm.getConstraintsInfo(model))
+ # beta constraints should always be satisfied
+ assert coefs["x1"] <= 1.5 or abs(1.5-coefs["x1"])<1e-6, "beta constraint x1 ({0}) >= 1.5 is violated.".format(coefs["x1"])
+
+if __name__ == "__main__":
+ pyunit_utils.standalone_test(test_bad_lambda_specification)
+else:
+ test_bad_lambda_specification()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_test_large.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_test_large.py
new file mode 100644
index 000000000000..dc337ae33cf6
--- /dev/null
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_16312_contrained_GLM_test_large.py
@@ -0,0 +1,108 @@
+import h2o
+from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm
+from tests import pyunit_utils
+import numpy as np
+import pandas as pd
+
+def data_prep(seed):
+ np.random.seed(seed)
+ x1 = np.random.normal(0, 10, 100000)
+ x2 = np.random.normal(10, 100 , 100000)
+ x3 = np.random.normal(20, 200, 100000)
+ x4 = np.random.normal(30, 3000, 100000)
+ x5 = np.random.normal(400, 4000, 100000)
+
+ y_raw = np.sin(x1)*100 + np.sin(x2)*100 + x3/20 + x3/30 + x5/400
+ y = np.random.normal(y_raw, 20)
+
+ data = {
+ 'x1': x1,
+ 'x2': x2,
+ 'x3': x3,
+ 'x4': x4,
+ 'x5': x5,
+ 'y': y,
+ }
+ return h2o.H2OFrame(pd.DataFrame(data))
+
+def test_bad_linear_constraints():
+ train_data = data_prep(123)
+ family = 'gaussian'
+ link = 'identity'
+ nfolds = 0
+ lambda_ = 0
+ seed = 1234
+ calc_like = True
+ compute_p_values = True
+ solver = 'irlsm'
+ predictors = ['x1', 'x2', 'x3', 'x4', 'x5']
+ response = "y"
+
+ linear_constraints2 = []
+
+ name = "x2"
+ values = 1
+ types = "Equal"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "x3"
+ values = 1
+ types = "Equal"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 0
+ types = "Equal"
+ contraint_numbers = 0
+ linear_constraints2.append([name, values, types, contraint_numbers])
+
+
+ linear_constraints = h2o.H2OFrame(linear_constraints2)
+ linear_constraints.set_names(["names", "values", "types", "constraint_numbers"])
+
+ params3 = {
+ "family" : family,
+ "link": link,
+ "lambda_" : lambda_,
+ "seed" : seed,
+ "nfolds" : nfolds,
+ "compute_p_values" : compute_p_values,
+ "calc_like" : calc_like,
+ "solver" : solver,
+ "linear_constraints": linear_constraints,
+ "standardize": True,
+ }
+
+ glm3 = glm(**params3)
+ glm3.train(x = predictors, y = response, training_frame = train_data)
+ print(glm.getConstraintsInfo(glm3))
+ coef3 = glm3.coef()
+ print(glm3.coef())
+
+ params2 = {
+ "family" : family,
+ "link": link,
+ "lambda_" : lambda_,
+ "seed" : seed,
+ "nfolds" : nfolds,
+ "compute_p_values" : compute_p_values,
+ "calc_like" : calc_like,
+ "solver" : solver
+ }
+ glm2 = glm(**params2)
+ glm2.train(x = predictors, y = response, training_frame = train_data)
+ print("Models built without linear constraints")
+ coef2 = glm2.coef()
+ print(coef2)
+ print("x2 + x3: {0}".format(coef2["x2"]+coef2["x3"]))
+
+ # check that model with constraints are closer to the constraints than models without constraints
+ assert (coef3["x2"]+coef3["x3"])<(coef2["x2"]+coef3["x3"]), \
+ "models built with constraints should be closer to the constraints x2+x3 but is not."
+
+if __name__ == "__main__":
+ pyunit_utils.standalone_test(test_bad_linear_constraints)
+else:
+ test_bad_linear_constraints()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py
index 4c01ed1f67d7..569a4304268b 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py
@@ -195,13 +195,13 @@ def test_constraints_binomial():
print(glm.getConstraintsInfo(h2o_glm_default_init))
- assert abs(logloss-init_logloss)<2e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \
+ assert abs(logloss-init_logloss)<1e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \
"and initialized with optimal GLM {1} should equal but is not." \
"".format(logloss, init_logloss)
- assert logloss <= init_random_logloss, "logloss from optimal GLM {0} should be less than GLM with constraints " \
+ assert abs(logloss-init_random_logloss)<1e-6, "logloss from optimal GLM {0} should be close to GLM with constraints " \
"and with random initial coefficients {1} but is" \
" not.".format(logloss, init_random_logloss)
- assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be less than GLM with constraints " \
+ assert abs(logloss-default_init_logloss)<1e-6, "logloss from optimal GLM {0} should be close to GLM with constraints " \
"and with default initial coefficients {1} but is" \
" not.".format(logloss, default_init_logloss)
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py
index 8a47d3773279..8c29822f0b87 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_constraints_binomial():
+def test_equality_constraints_only_binomial():
'''
This test checks and make sure the equality constraints work with binomial family. Coefficients are initialized
with glm coefficients built without contraints, default coefficients and random coefficients.
@@ -124,11 +124,12 @@ def test_constraints_binomial():
"".format(default_init_logloss, h2o_glm_default_init._model_json["output"]["model_summary"].cell_values[0][6]))
print(glm.getConstraintsInfo(h2o_glm_default_init))
- assert init_random_logloss >= logloss, "Random initialization logloss with constraints should be worst than GLM " \
- "without constraints but is not."
+ assert abs(init_random_logloss - logloss) < 1e-6, \
+ "Random initialization logloss {0} with constraints should be similary to than GLM without constraints {1} but" \
+ " is not.".format(init_random_logloss, logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_constraints_binomial)
+ pyunit_utils.standalone_test(test_equality_constraints_only_binomial)
else:
- test_constraints_binomial()
+ test_equality_constraints_only_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py
index acf74d648dea..dd884aaea48b 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_constraints_binomial():
+def test_equality_linear_constraints_binomial():
'''
This test checks and make sure the equality constraints work with binomial family. Coefficients are initialized
with glm coefficients built without constraints, default coefficients and random coefficients. Note in this case,
@@ -156,18 +156,18 @@ def test_constraints_binomial():
" taken to build the model: {1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init)))
print(glm.getConstraintsInfo(h2o_glm_default_init))
- assert abs(logloss-init_logloss)<2e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \
+ assert abs(logloss-init_logloss)<1e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \
"and initialized with optimal GLM {1} should equal but is not." \
"".format(logloss, init_logloss)
- assert logloss<=init_random_logloss, "logloss from optimal GLM {0} should be lower than GLM with constraints " \
+ assert abs(logloss-init_random_logloss)<1e-6, "logloss from optimal GLM {0} should be close to GLM with constraints " \
"and with random initial coefficients {1} but is" \
" not.".format(logloss, init_random_logloss)
- assert logloss<=default_init_logloss, "logloss from optimal GLM {0} should be less than GLM with constraints " \
+ assert abs(logloss-default_init_logloss)<1e-6, "logloss from optimal GLM {0} should be close to GLM with constraints " \
"and with default initial coefficients {1} but is" \
" not.".format(logloss, default_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_constraints_binomial)
+ pyunit_utils.standalone_test(test_equality_linear_constraints_binomial)
else:
- test_constraints_binomial()
+ test_equality_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py
index 97ddf83bbe29..732bdda9d8e5 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_light_tight_linear_constraints_only_gaussian():
+def test_light_tight_linear_constraints_binomial():
'''
Test constrained GLM with beta, equality and less than and equal to constraints. The constraints are not very
tight. However, coefficients from GLM built without constraints won't be able to satisfied the constraints.
@@ -207,6 +207,6 @@ def test_light_tight_linear_constraints_only_gaussian():
"not.".format(logloss, random_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian)
+ pyunit_utils.standalone_test(test_light_tight_linear_constraints_binomial)
else:
- test_light_tight_linear_constraints_only_gaussian()
+ test_light_tight_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py
index 96146d98234a..8e50603e9f15 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_light_tight_linear_constraints_only_gaussian():
+def test_light_tight_linear_constraints_only_binomial():
'''
Test constrained GLM with equality and less than and equal to constraints. The constraints are not very
tight. However, coefficients from GLM built without constraints won't be able to satisfied the constraints.
@@ -173,7 +173,7 @@ def test_light_tight_linear_constraints_only_gaussian():
"{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init)))
print(glm.getConstraintsInfo(h2o_glm_random_init))
- assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
+ assert abs(logloss - optimal_init_logloss)<1e-6, "logloss from optimal GLM {0} should be close to logloss from GLM with light tight" \
" constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss)
assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
@@ -185,6 +185,6 @@ def test_light_tight_linear_constraints_only_gaussian():
"not.".format(logloss, random_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian)
+ pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_binomial)
else:
- test_light_tight_linear_constraints_only_gaussian()
+ test_light_tight_linear_constraints_only_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py
index 103086f4ca6f..d59c80fd99a5 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_light_tight_linear_constraints_only_gaussian():
+def test_light_tight_linear_constraints_only_binomial():
'''
Test constrained GLM with less than and equal to constraints. The constraints are not very
tight. However, coefficients from GLM built without constraints won't be able to satisfied the constraints.
@@ -189,7 +189,7 @@ def test_light_tight_linear_constraints_only_gaussian():
print(glm.getConstraintsInfo(h2o_glm_random_init))
print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init)))
- assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
+ assert abs(logloss - optimal_init_logloss)<1e-6, "logloss from optimal GLM {0} should be close to logloss from GLM with light tight" \
" constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss)
assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
@@ -201,6 +201,6 @@ def test_light_tight_linear_constraints_only_gaussian():
"not.".format(logloss, random_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian)
+ pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_binomial)
else:
- test_light_tight_linear_constraints_only_gaussian()
+ test_light_tight_linear_constraints_only_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py
index 3a75c301cc65..08778437db07 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_constraints_binomial():
+def test_loose_beta_linear_constraints_binomial():
'''
check and make sure coefficients close to GLM built without constraints are generated with loose constraints
that are satisfied with coefficients from GLM without constraints. Only beta and less than and equal to
@@ -157,6 +157,6 @@ def test_constraints_binomial():
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_constraints_binomial)
+ pyunit_utils.standalone_test(test_loose_beta_linear_constraints_binomial)
else:
- test_constraints_binomial()
+ test_loose_beta_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py
index af40e6503b38..f9b0d17f976d 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_constraints_binomial():
+def test_loose_linear_constraints_binomial():
'''
check and make sure coefficients close to GLM built without constraints are generated with loose constraints
that are satisfied with coefficients from GLM without constraints. Only less than and equal to
@@ -135,6 +135,6 @@ def test_constraints_binomial():
" but is not.".format(logloss, default_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_constraints_binomial)
+ pyunit_utils.standalone_test(test_loose_linear_constraints_binomial)
else:
- test_constraints_binomial()
+ test_loose_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py
index 56a9c052625b..81fd336e2603 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py
@@ -198,7 +198,7 @@ def test_redundant_constraints():
except Exception as ex:
print(ex)
temp = str(ex)
- assert ("redundant and possibly conflicting linear constraints" in temp), "Wrong exception was received."
+ assert ("redundant linear constraints:" in temp), "Wrong exception was received."
print("redundant constraint test passed!")
if __name__ == "__main__":
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py
index d7befd7da108..f40887389093 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_light_tight_linear_constraints_only_gaussian():
+def test_tight_beta_linear_constraints_binomial():
'''
Test constrained GLM with beta, equality and less than and equal to constraints. The constraints are very
tight and coefficients from GLM built without constraints won't be able to satisfied the constraints.
@@ -174,42 +174,6 @@ def test_light_tight_linear_constraints_only_gaussian():
types = "Equal"
contraint_numbers = 5
tight_constraints.append([name, values, types, contraint_numbers])
-
- name = "C19"
- values = 0.5
- types = "Equal"
- contraint_numbers = 4
- tight_constraints.append([name, values, types, contraint_numbers])
-
- name = "C10.1"
- values = -0.3
- types = "Equal"
- contraint_numbers = 4
- tight_constraints.append([name, values, types, contraint_numbers])
-
- name = "constant"
- values = -0.5
- types = "Equal"
- contraint_numbers = 4
- tight_constraints.append([name, values, types, contraint_numbers])
-
- name = "C18"
- values = 0.75
- types = "Equal"
- contraint_numbers = 5
- tight_constraints.append([name, values, types, contraint_numbers])
-
- name = "C20"
- values = -0.13
- types = "Equal"
- contraint_numbers = 5
- tight_constraints.append([name, values, types, contraint_numbers])
-
- name = "constant"
- values = -3
- types = "Equal"
- contraint_numbers = 5
- tight_constraints.append([name, values, types, contraint_numbers])
linear_constraints2 = h2o.H2OFrame(tight_constraints)
linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"])
@@ -317,6 +281,6 @@ def test_light_tight_linear_constraints_only_gaussian():
"not.".format(logloss, random_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian)
+ pyunit_utils.standalone_test(test_tight_beta_linear_constraints_binomial)
else:
- test_light_tight_linear_constraints_only_gaussian()
+ test_tight_beta_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py
index 1f4888b195a3..94ac1155c494 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_light_tight_linear_constraints_only_gaussian():
+def test_tight_equality_linear_constraints_binomial():
'''
Test constrained GLM with equality and less than and equal to constraints. The constraints are very
tight and coefficients from GLM built without constraints won't be able to satisfied the constraints.
@@ -225,7 +225,7 @@ def test_light_tight_linear_constraints_only_gaussian():
print(glm.getConstraintsInfo(h2o_glm_random_init))
print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init)))
- assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
+ assert abs(logloss - optimal_init_logloss)<1e-6, "logloss from optimal GLM {0} should be close to logloss from GLM with light tight" \
" constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss)
assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
@@ -237,6 +237,6 @@ def test_light_tight_linear_constraints_only_gaussian():
"not.".format(logloss, random_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian)
+ pyunit_utils.standalone_test(test_tight_equality_linear_constraints_binomial)
else:
- test_light_tight_linear_constraints_only_gaussian()
+ test_tight_equality_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py
index d8668bad9776..cc5b5385c8d3 100644
--- a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py
@@ -3,7 +3,7 @@
from tests import pyunit_utils
from tests.pyunit_utils import utils_for_glm_tests
-def test_light_tight_linear_constraints_only_gaussian():
+def test_tight_linear_constraints_binomial():
'''
Test constrained GLM with less than and equal to constraints. The constraints are very
tight and coefficients from GLM built without constraints won't be able to satisfied the constraints.
@@ -189,7 +189,7 @@ def test_light_tight_linear_constraints_only_gaussian():
print(glm.getConstraintsInfo(h2o_glm_random_init))
print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init)))
- assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
+ assert abs(logloss - optimal_init_logloss)<1e-6, "logloss from optimal GLM {0} should be close to logloss from GLM with light tight" \
" constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss)
assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \
@@ -201,6 +201,6 @@ def test_light_tight_linear_constraints_only_gaussian():
"not.".format(logloss, random_init_logloss)
if __name__ == "__main__":
- pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian)
+ pyunit_utils.standalone_test(test_tight_linear_constraints_binomial)
else:
- test_light_tight_linear_constraints_only_gaussian()
+ test_tight_linear_constraints_binomial()
diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_gh_16203_constrained_glm_example.py b/h2o-py/tests/testdir_algos/glm/pyunit_gh_16203_constrained_glm_example.py
new file mode 100644
index 000000000000..86ecfb4236f0
--- /dev/null
+++ b/h2o-py/tests/testdir_algos/glm/pyunit_gh_16203_constrained_glm_example.py
@@ -0,0 +1,206 @@
+import h2o
+from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm
+from tests import pyunit_utils
+
+def test_constrained_glm_example():
+ '''
+ Simple example to showcase how to call constrained GLM.
+ '''
+ #train = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/glm_test/binomial_20_cols_10KRows.csv")
+ train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv"))
+ for ind in range(10):
+ train[ind] = train[ind].asfactor()
+ train["C21"] = train["C21"].asfactor()
+ response = "C21"
+ predictors = list(range(0,20))
+ # add beta constraints
+ bc = []
+ name = "C11"
+ lower_bound = -3.5
+ upper_bound = 0
+ bc.append([name, lower_bound, upper_bound])
+
+ name = "C18"
+ lower_bound = 6
+ upper_bound = 7
+ bc.append([name, lower_bound, upper_bound])
+
+ name = "C15"
+ lower_bound = -9
+ upper_bound = -6
+ bc.append([name, lower_bound, upper_bound])
+
+ name = "C16"
+ lower_bound = -20
+ upper_bound = -10
+ bc.append([name, lower_bound, upper_bound])
+
+ beta_constraints = h2o.H2OFrame(bc)
+ beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"])
+
+ tight_constraints = [] # this constraint is satisfied by default coefficient initialization
+
+ # add tight constraints
+ name = "C1.1"
+ values = 0.5
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C2.1"
+ values = -0.25
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = -1
+ types = "LessThanEqual"
+ contraint_numbers = 0
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C4.1"
+ values = 1.5
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C17"
+ values = 3
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C15"
+ values = -2
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = -5
+ types = "LessThanEqual"
+ contraint_numbers = 1
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C12"
+ values = -0.5
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C13"
+ values = -1.5
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C14"
+ values = 2
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = -3
+ types = "LessThanEqual"
+ contraint_numbers = 2
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C11"
+ values = 0.25
+ types = "LessThanEqual"
+ contraint_numbers = 3
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C18"
+ values = -0.5
+ types = "LessThanEqual"
+ contraint_numbers = 3
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C19"
+ values = 0.75
+ types = "LessThanEqual"
+ contraint_numbers = 3
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = 5
+ types = "LessThanEqual"
+ contraint_numbers = 3
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C19"
+ values = 0.5
+ types = "Equal"
+ contraint_numbers = 4
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C10.1"
+ values = -0.3
+ types = "Equal"
+ contraint_numbers = 4
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = -0.25
+ types = "Equal"
+ contraint_numbers = 4
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C18"
+ values = 0.75
+ types = "Equal"
+ contraint_numbers = 5
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "C20"
+ values = -0.13
+ types = "Equal"
+ contraint_numbers = 5
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ name = "constant"
+ values = -1.5
+ types = "Equal"
+ contraint_numbers = 5
+ tight_constraints.append([name, values, types, contraint_numbers])
+
+ linear_constraints2 = h2o.H2OFrame(tight_constraints)
+ linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"])
+
+ random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453,
+ 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713,
+ 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719,
+ 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326,
+ 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427,
+ 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426,
+ 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668,
+ 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405,
+ 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243,
+ 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204,
+ 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784,
+ 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103,
+ 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289,
+ 0.224690851359456, 0.5809304720756304, 0.36863807988348585]
+ params = {"family":"binomial", "lambda_":0.0, "seed":12345, "remove_collinear_columns":True, "solver":"IRLSM",
+ "linear_constraints":linear_constraints2, "beta_constraints":beta_constraints}
+
+ # build constrained GLM with default coefficient initialization, all coefficients zero except intercept
+ constrained_glm_default_init = glm(**params)
+ constrained_glm_default_init.train(x=predictors, y=response, training_frame=train)
+ logloss_default = constrained_glm_default_init.model_performance()._metric_json['logloss']
+ # build constrained GLM model with random coefficient initialization
+ params["startval"] = random_coef
+ constrained_glm_random_init = glm(**params)
+ constrained_glm_random_init.train(x=predictors, y=response, training_frame=train)
+ logloss = constrained_glm_random_init.model_performance()._metric_json['logloss']
+
+ assert logloss_default >= logloss or abs(logloss_default-logloss) < 1e-2
+
+
+if __name__ == "__main__":
+ pyunit_utils.standalone_test(test_constrained_glm_example)
+else:
+ test_constrained_glm_example()
diff --git a/h2o-py/tests/testdir_algos/modelselection/pyunit_PUBDEV_8675_modelselection_fail.py b/h2o-py/tests/testdir_algos/modelselection/pyunit_PUBDEV_8675_modelselection_fail.py
index 7540abf1dc5c..186062fba82f 100644
--- a/h2o-py/tests/testdir_algos/modelselection/pyunit_PUBDEV_8675_modelselection_fail.py
+++ b/h2o-py/tests/testdir_algos/modelselection/pyunit_PUBDEV_8675_modelselection_fail.py
@@ -8,7 +8,8 @@
from h2o.estimators.glm import H2OGeneralizedLinearEstimator
# Megan Kurka found that categorical columns do not work with modelselection backward mode. I fixed the bug and
-# extended her test to check that each time a predictor is dropped, it must has the smallest z-value magnitude.
+# extended her test to check that each time a predictor is dropped, the best performing level is compared to other
+# predictors. If the best level is not good enough, the whole enum predictor is dropped.
def test_megan_failure():
df = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/demos/bank-additional-full.csv")
y = "y"
@@ -28,7 +29,6 @@ def test_megan_failure():
best_predictor_subset = backward_model.get_best_model_predictors()
counter = 0
- back_coef = backward_model.coef()
for ind in list(range(num_models-1, 0, -1)):
pred_large = coefficient_orders[ind]
pred_small = coefficient_orders[ind-1]
@@ -40,11 +40,11 @@ def test_megan_failure():
# assert z-values removed has smallest magnitude
x = best_predictor_subset[ind]
- assert_smallest_z_removed(back_coef[ind], z_values_list, z_values_removed, pred_large, predictor_removed, x, y, df)
+ assert_correct_z_removed(z_values_list, z_values_removed, pred_large, predictor_removed, x, y, df)
counter += 1
-def assert_smallest_z_removed(back_coef, z_values_backward, z_values_removed, coeff_backward, predictor_removed, x, y, df):
+def assert_correct_z_removed(z_values_backward, z_values_removed, coeff_backward, predictor_removed, x, y, df):
glm_model = H2OGeneralizedLinearEstimator(seed=1234, remove_collinear_columns=True, lambda_=0.0, compute_p_values=True)
glm_model.train(x=x, y=y, training_frame=df)
cat_predictors = extractCatCols(df, x)
@@ -53,11 +53,21 @@ def assert_smallest_z_removed(back_coef, z_values_backward, z_values_removed, co
model_z_values = glm_model._model_json["output"]["coefficients_table"]["z_value"]
model_coeffs = glm_model._model_json["output"]["coefficients_table"]["names"]
- assert_equal_z_values(back_coef, glm_model.coef(), z_values_backward, coeff_backward, model_z_values, model_coeffs)
- min_z_value = min(z_values_removed)
+ assert_equal_z_values(z_values_backward, coeff_backward, model_z_values, model_coeffs)
+
+ num_predictor_removed = False
+ for one_value in predictor_removed:
+ if one_value in num_predictors:
+ num_predictor_removed = True
+ break
+ if num_predictor_removed:
+ min_z_value = min(z_values_removed)
+ else:
+ min_z_value = max(z_values_removed)
+
# check that predictor with smallest z-value magnitude is removed
- assert_smallest_z_value_numerical(num_predictors, min_z_value, model_coeffs, model_z_values)
- assert_smallest_z_value_categorical(cat_predictors, min_z_value, model_coeffs, model_z_values)
+ assert_correct_z_value_numerical(num_predictors, min_z_value, model_coeffs, model_z_values)
+ assert_correct_z_value_categorical(cat_predictors, min_z_value, model_coeffs, model_z_values)
for name in cat_predictors:
for coeff_name in predictor_removed:
@@ -66,7 +76,7 @@ def assert_smallest_z_removed(back_coef, z_values_backward, z_values_removed, co
return
x.remove(predictor_removed[0]) # numerical predictor is removed
-def assert_smallest_z_value_categorical(cat_predictors, min_z_value, model_coeffs, model_z_values):
+def assert_correct_z_value_categorical(cat_predictors, min_z_value, model_coeffs, model_z_values):
for name in cat_predictors:
model_z = []
for coeff_name in model_coeffs:
@@ -80,7 +90,7 @@ def assert_smallest_z_value_categorical(cat_predictors, min_z_value, model_coeff
"than mininum_z_values {2}".format(name, model_z, min_z_value)
-def assert_smallest_z_value_numerical(num_predictors, min_z_value, model_coeffs, model_z_values):
+def assert_correct_z_value_numerical(num_predictors, min_z_value, model_coeffs, model_z_values):
for name in num_predictors:
pred_ind = model_coeffs.index(name)
val = model_z_values[pred_ind]
@@ -96,7 +106,7 @@ def extractCatCols(df, x):
cat_pred.append(name)
return cat_pred
-def assert_equal_z_values(back_coef, curr_coef, z_values_backward, coeff_backward, model_z_values, glm_coeff):
+def assert_equal_z_values(z_values_backward, coeff_backward, model_z_values, glm_coeff):
for coeff in glm_coeff:
backward_z_value = z_values_backward[coeff_backward.index(coeff)]
model_z_value = model_z_values[glm_coeff.index(coeff)]
diff --git a/h2o-py/tests/testdir_algos/word2vec/pyunit_text8_word2vec_large.py b/h2o-py/tests/testdir_algos/word2vec/pyunit_text8_word2vec_large.py
index 3d030e349a1e..de79c21c063d 100644
--- a/h2o-py/tests/testdir_algos/word2vec/pyunit_text8_word2vec_large.py
+++ b/h2o-py/tests/testdir_algos/word2vec/pyunit_text8_word2vec_large.py
@@ -14,11 +14,16 @@ def word2vec():
w2v_model = H2OWord2vecEstimator(epochs=1, word_model=word_model)
w2v_model.train(training_frame=train)
- synonyms = w2v_model.find_synonyms("horse", 3)
+ cnt = 10
+ synonyms = w2v_model.find_synonyms("horse", cnt)
print(synonyms)
-
- assert len(synonyms) == 3, "there should be three synonmys"
-
+ assert len(synonyms) == cnt, "There should be ten synonyms."
+
+ # GH-16192 find_synonyms returns empty dataset if there is no synonyms to find
+ synonyms = w2v_model.find_synonyms("hhorse", cnt)
+ print(synonyms)
+ assert len(synonyms) == 0, "There should be zero synonyms."
+
if __name__ == "__main__":
pyunit_utils.standalone_test(word2vec)
diff --git a/h2o-py/tests/testdir_apis/Data_Manipulation/pyunit_h2oH2OFrame.py b/h2o-py/tests/testdir_apis/Data_Manipulation/pyunit_h2oH2OFrame.py
index b9f101dda2a7..b284f6e53f39 100644
--- a/h2o-py/tests/testdir_apis/Data_Manipulation/pyunit_h2oH2OFrame.py
+++ b/h2o-py/tests/testdir_apis/Data_Manipulation/pyunit_h2oH2OFrame.py
@@ -125,12 +125,10 @@ def H2OFrame_from_H2OFrame():
assert dupl4.columns == ["n1", "s1"]
-def H2OFrame_skipped_columns_is_BUGGY():
- try:
- h2o.H2OFrame(data, skipped_columns=[1])
- assert False, "skipped_columns handling may be fixed now" # parse_setup is absolutely weird, with only half parameters passed to build the ParseSetup, and then a bunch of logic done locally, that's why it's buggy: see issue https://github.com/h2oai/h2o-3/issues/15947
- except ValueError as e:
- assert "length of col_names should be equal to the number of columns parsed: 4 vs 3" in str(e)
+def H2OFrame_skipped_columns_BUG_fixed():
+ f1 = h2o.H2OFrame(data, skipped_columns=[1])
+ f2 = h2o.H2OFrame(data)
+ assert f1.ncol == (f2.ncol-1), "expected number of columns: {0}, actual column numbers: {1}".format(f1.ncol, (f2.ncol-1))
pu.run_tests([
@@ -141,5 +139,5 @@ def H2OFrame_skipped_columns_is_BUGGY():
H2OFrame_from_pandas,
H2OFrame_from_scipy,
H2OFrame_from_H2OFrame,
- H2OFrame_skipped_columns_is_BUGGY
+ H2OFrame_skipped_columns_BUG_fixed
])
diff --git a/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2oget_timezone_DEPRECATED.py b/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2oget_timezone_DEPRECATED.py
index ae4c5e76eeba..ecd499ee01a7 100644
--- a/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2oget_timezone_DEPRECATED.py
+++ b/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2oget_timezone_DEPRECATED.py
@@ -19,7 +19,7 @@ def h2oget_timezone():
timezones = h2o.list_timezones()
assert_is_type(timezones, H2OFrame)
- assert timezones.nrow == 467, "h2o.get_timezone() returns frame with wrong row number."
+ assert timezones.nrow == 459, "h2o.get_timezone() returns frame with wrong row number."
assert timezones.ncol == 1, "h2o.get_timezone() returns frame with wrong column number."
if __name__ == "__main__":
diff --git a/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2olist_timezones_DEPRECATED.py b/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2olist_timezones_DEPRECATED.py
index c6ae86359813..d5330ff20cd3 100644
--- a/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2olist_timezones_DEPRECATED.py
+++ b/h2o-py/tests/testdir_apis/H2O_Module/pyunit_h2olist_timezones_DEPRECATED.py
@@ -14,7 +14,7 @@ def h2olist_timezones():
timezones = h2o.list_timezones()
assert_is_type(timezones, H2OFrame)
- assert timezones.nrow == 467, "h2o.get_timezone() returns frame with wrong row number."
+ assert timezones.nrow == 459, "h2o.get_timezone() returns frame with wrong row number."
assert timezones.ncol == 1, "h2o.get_timezone() returns frame with wrong column number."
diff --git a/h2o-py/tests/testdir_misc/pyunit_cluster.py b/h2o-py/tests/testdir_misc/pyunit_cluster.py
index 2424170ef1d6..9c367484a27f 100644
--- a/h2o-py/tests/testdir_misc/pyunit_cluster.py
+++ b/h2o-py/tests/testdir_misc/pyunit_cluster.py
@@ -11,9 +11,9 @@ def test_cluster_status():
def test_cluster_properties():
cl = h2o.cluster()
- assert len(cl._schema_attrs_) == 24
+ assert len(cl._schema_attrs_) == 25
for k in cl._schema_attrs_.keys():
- assert getattr(cl, k) is not None
+ assert getattr(cl, k) is not None or k == "web_ip"
def test_exception_on_unknown_cluster_property():
diff --git a/h2o-py/tests/testdir_misc/pyunit_export_parquet_npe.py b/h2o-py/tests/testdir_misc/pyunit_export_parquet_npe.py
new file mode 100644
index 000000000000..1a8c8fb306e7
--- /dev/null
+++ b/h2o-py/tests/testdir_misc/pyunit_export_parquet_npe.py
@@ -0,0 +1,23 @@
+import sys
+import tempfile
+
+sys.path.insert(1, "../../../")
+import h2o
+from tests import pyunit_utils
+
+
+def test_export_file_npe_gh_16161():
+ with tempfile.TemporaryDirectory() as dir:
+ df = h2o.create_frame(rows=100, cols=10, string_fraction=0.1, seed=5, seed_for_column_types=25)
+ h2o.export_file(df, path=dir, format="parquet", write_checksum=False)
+ df2 = h2o.import_file(dir)
+ assert pyunit_utils.compare_frames(df, df2, tol_numeric=1e-10, numElements=0)
+
+
+if __name__ == "__main__":
+ pyunit_utils.standalone_test(test_export_file_npe_gh_16161)
+else:
+ test_export_file_npe_gh_16161()
+
+
+
diff --git a/h2o-py/tests/testdir_misc/pyunit_export_zstd.py b/h2o-py/tests/testdir_misc/pyunit_export_zstd.py
new file mode 100644
index 000000000000..e3704c2cb11a
--- /dev/null
+++ b/h2o-py/tests/testdir_misc/pyunit_export_zstd.py
@@ -0,0 +1,37 @@
+import sys
+sys.path.insert(1,"../../../")
+import h2o
+from tests import pyunit_utils
+from os import path
+import struct
+
+'''
+Export file with h2o.export_file compressed with 'zstd'
+'''
+
+
+def is_zstd_file(path):
+ with open(path, 'rb') as f:
+ magic_bytes = f.read(4)
+ return struct.unpack('