Merge branch 'jun_devel' of https://github.com/usc-isi-i2/dsbox-primi…

…tives
usc-isi-i2 · Feb 19, 2020 · a5a1819 · a5a1819
2 parents 95202d2 + 390595a
commit a5a1819
Show file tree

Hide file tree

Showing 46 changed files with 215,171 additions and 636 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,15 +1,15 @@
 dist: xenial
 sudo: required
 language: python
-cache: pip
 python:
-- 3.6.6
+- 3.6.8
 before_install:
 - echo -e "machine github.com\n  login $GH_TOKEN" > ~/.netrc
 install:
 - echo "Installing dependencies"
 - sudo apt-get update
-- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+- sudo apt-get install unzip
+- wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh --no-verbose
 - bash miniconda.sh -b -p $HOME/miniconda
 - export PATH="$HOME/miniconda/bin:$PATH"
 - hash -r

diff --git a/.travis/README.md b/.travis/README.md
@@ -0,0 +1,15 @@
+## Dsbox Primitive Unit Test
+#### Test pipeline
+How To add more testing pipeline (e.g. for new primitive):
+1. In `library.py`, following the existed format (like `DefaultClassificationTemplate`) to create a new `Template` class.
+2. If the added new pipeline is for new `runType` (you can also check it to ensure). Add the corresponding mapping In dict `DATASET_MAPPER` at line 416 on file `template.py`, follow the format. Ensure it is correct, otherwise the system would failed on finding correct dataset to run.
+3. ~~Go to `generate-pipelines-json.py` and add the new class for the import (line 8) part.~~ The system should now import all templates.
+4. Add it to `TEMPLATE_LIST` on `generate-pipelines-json.py`.
+5. Then, the unit test system will automatically run the new template and generate corresponding `pipeline.json` file that can used to upload as sample pipeline.
+
+#### primitives that do not have pipelines now
+1. data preprocessing: `label_encoder`, `greedy_imputation`, `multitable_featurization`
+2. `column_fold` and `unfold`
+3. Video classification: `LSTM`, `inceptionV3`, 
+4. concat related: `horizontal concat`,
+5. Dataset splitter: `splitter`
diff --git a/.travis/generate-pipelines-json.py b/.travis/generate-pipelines-json.py
diff --git a/.travis/library.py b/.travis/library.py
diff --git a/.travis/pip_install.sh b/.travis/pip_install.sh
@@ -1,10 +1,19 @@
 #!/bin/bash
-pip install -e git+https://gitlab.com/datadrivendiscovery/d3m@a8af7585fdd85e2218ca88b257bb0ec71adabfb3#egg=d3m --progress-bar off
-pip install -e git+https://gitlab.com/datadrivendiscovery/common-primitives.git@5c43e65d306a4f36d53db2fb497c9869e2fb7294#egg=common_primitives --progress-bar off
-pip install -e git+https://gitlab.com/datadrivendiscovery/sklearn-wrap@dist#egg=sklearn-wrap --progress-bar off
+pip install -e git+https://gitlab.com/datadrivendiscovery/d3m@be853095932d4a94bea45da61192a926bfcb1dbd#egg=d3m --progress-bar off
+pip install -e git+https://gitlab.com/datadrivendiscovery/common-primitives.git@15e84bff9b310068c071d79b255f3314df183466#egg=common_primitives --progress-bar off
+pip install -e git+https://gitlab.com/datadrivendiscovery/sklearn-wrap@4a2cfd1dc749bb13ce807b2bf2436a45cd49c695#egg=sklearn-wrap --progress-bar off
 pip uninstall -y tensorflow-gpu
 export LD_LIBRARY_PATH="$HOME/miniconda/envs/ta1-test-env/lib:$LD_LIBRARY_PATH"
-pip install tensorflow==1.12.0
+pip install tensorflow==2.0.0
 pip install -e . --progress-bar off
-pip install -e git+https://github.com/brekelma/dsbox_corex@master#egg=dsbox_corex --progress-bar off
+pip install -e git+https://github.com/brekelma/dsbox_corex@5ebdd6ee66aa5ddb48e3c97d98145586d95c9c1e#egg=dsbox_corex --progress-bar off
 pip list
+wget https://github.com/fchollet/deep-learning-models/releases/download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5 --no-verbose
+mv resnet50_weights_tf_dim_ordering_tf_kernels.h5 bdc6c9f787f9f51dffd50d895f86e469cc0eb8ba95fd61f0801b1a264acb4819
+wget https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5 --no-verbose
+mv vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5 bfe5187d0a272bed55ba430631598124cff8e880b98d38c9e56c8d66032abdc1
+wget https://pjreddie.com/media/files/yolov3.weights --no-verbose
+mv yolov3.weights 523e4e69e1d015393a1b0a441cef1d9c7659e3eb2d7e15f793f060a21b32f297
+wget https://github.com/keras-team/keras-applications/releases/download/resnet/resnet50_weights_tf_dim_ordering_tf_kernels.h5 --no-verbose
+mv resnet50_weights_tf_dim_ordering_tf_kernels.h5 7011d39ea4f61f4ddb8da99c4addf3fae4209bfda7828adb4698b16283258fbe
+ls -l
diff --git a/.travis/pipeline_configs/.DS_Store b/.travis/pipeline_configs/.DS_Store
diff --git a/.travis/pipeline_configs/default_classification.py b/.travis/pipeline_configs/default_classification.py
diff --git a/.travis/pre_ran_pipelines/LL1_tidy/pp_object_detection.json b/.travis/pre_ran_pipelines/LL1_tidy/pp_object_detection.json
@@ -0,0 +1,157 @@
+{
+  "id": "dd2d98ed-5d94-4245-a0c9-0861ed7bc177",
+  "schema": "https://metadata.datadrivendiscovery.org/schemas/v0/pipeline.json",
+  "created": "2020-01-24T01:01:01.853055Z",
+  "inputs": [
+    {
+      "name": "input dataset"
+    }
+  ],
+  "outputs": [
+    {
+      "data": "steps.4.produce",
+      "name": "predictions of input dataset"
+    }
+  ],
+  "steps": [
+    {
+      "type": "PRIMITIVE",
+      "primitive": {
+        "id": "f31f8c1f-d1c5-43e5-a4b2-2ae4a761ef2e",
+        "version": "0.2.0",
+        "python_path": "d3m.primitives.data_transformation.denormalize.Common",
+        "name": "Denormalize datasets",
+        "digest": "5ac405757790f53ed8bfdf782ea5805c3d115dca1df1d1479c6478c6d3038340"
+      },
+      "arguments": {
+        "inputs": {
+          "type": "CONTAINER",
+          "data": "inputs.0"
+        }
+      },
+      "outputs": [
+        {
+          "id": "produce"
+        }
+      ]
+    },
+    {
+      "type": "PRIMITIVE",
+      "primitive": {
+        "id": "4b42ce1e-9b98-4a25-b68e-fad13311eb65",
+        "version": "0.3.0",
+        "python_path": "d3m.primitives.data_transformation.dataset_to_dataframe.Common",
+        "name": "Extract a DataFrame from a Dataset",
+        "digest": "422744651afd5995d029a227a1dd7b1696038816b7eb9601f37d661757812aee"
+      },
+      "arguments": {
+        "inputs": {
+          "type": "CONTAINER",
+          "data": "steps.0.produce"
+        }
+      },
+      "outputs": [
+        {
+          "id": "produce"
+        }
+      ]
+    },
+    {
+      "type": "PRIMITIVE",
+      "primitive": {
+        "id": "4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
+        "version": "0.3.0",
+        "python_path": "d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common",
+        "name": "Extracts columns by semantic type",
+        "digest": "30cceb9812b430d6550d54766b4f674b68b92531fc2ad63f56818ea002399c13"
+      },
+      "arguments": {
+        "inputs": {
+          "type": "CONTAINER",
+          "data": "steps.1.produce"
+        }
+      },
+      "outputs": [
+        {
+          "id": "produce"
+        }
+      ],
+      "hyperparams": {
+        "semantic_types": {
+          "type": "VALUE",
+          "data": [
+            "https://metadata.datadrivendiscovery.org/types/PrimaryMultiKey",
+            "https://metadata.datadrivendiscovery.org/types/FileName"
+          ]
+        }
+      }
+    },
+    {
+      "type": "PRIMITIVE",
+      "primitive": {
+        "id": "4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
+        "version": "0.3.0",
+        "python_path": "d3m.primitives.data_transformation.extract_columns_by_semantic_types.Common",
+        "name": "Extracts columns by semantic type",
+        "digest": "30cceb9812b430d6550d54766b4f674b68b92531fc2ad63f56818ea002399c13"
+      },
+      "arguments": {
+        "inputs": {
+          "type": "CONTAINER",
+          "data": "steps.1.produce"
+        }
+      },
+      "outputs": [
+        {
+          "id": "produce"
+        }
+      ],
+      "hyperparams": {
+        "semantic_types": {
+          "type": "VALUE",
+          "data": [
+            "https://metadata.datadrivendiscovery.org/types/TrueTarget"
+          ]
+        }
+      }
+    },
+    {
+      "type": "PRIMITIVE",
+      "primitive": {
+        "id": "dsbox-featurizer-object-detection-yolo",
+        "version": "1.5.3",
+        "python_path": "d3m.primitives.feature_extraction.yolo.DSBOX",
+        "name": "DSBox Object Detection YOLO",
+        "digest": "2db0c52b7bd9ae94ccfdae549f07a05b936113e59fd07a9ecc4318b5fc3067a2"
+      },
+      "arguments": {
+        "inputs": {
+          "type": "CONTAINER",
+          "data": "steps.2.produce"
+        },
+        "outputs": {
+          "type": "CONTAINER",
+          "data": "steps.3.produce"
+        }
+      },
+      "outputs": [
+        {
+          "id": "produce"
+        }
+      ],
+      "hyperparams": {
+        "epochs": {
+          "type": "VALUE",
+          "data": 200
+        },
+        "use_fitted_weight": {
+          "type": "VALUE",
+          "data": false
+        }
+      }
+    }
+  ],
+  "name": "DefaultObjectDetectionTemplate:140186136032384",
+  "description": "",
+  "digest": "862956d95719977f9b0cc485a8742eabb5b2a355b775f9214ea3037281c4d35f"
+}