Merge pull request #307 from h2oai/mojo_cpp_scorer

Mojo cpp scorer

aws-sagemaker-hosted-scorer-cpp/.gitignore

@@ -0,0 +1,8 @@
+.idea/
+.gradle/
+.vscode/
+build/
+dist/
+.cproject
+.project
+.settings/

aws-sagemaker-hosted-scorer-cpp/Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.8
+
+RUN apt-get update && \
+    apt-get install -y libopenblas-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt /tmp/requirements.txt
+
+RUN pip install pip==21.1 && pip install -r /tmp/requirements.txt
+
+ENV DRIVERLESS_AI_LICENSE_FILE='/opt/ml/model/license.sig'
+ENV MOJO_FILE_PATH='/opt/ml/model/pipeline.mojo'
+ENV WEB_SERVER_WORKERS=1
+
+RUN mkdir -p /opt/ml/code
+
+COPY py/scorer/mojo_cpp_scorer.py /opt/ml/code
+
+WORKDIR /opt/ml/code
+
+ENTRYPOINT gunicorn -w ${WEB_SERVER_WORKERS:-1} -b 0.0.0.0:8080 mojo_cpp_scorer:app

aws-sagemaker-hosted-scorer-cpp/README.md

@@ -0,0 +1,121 @@
+# DAI Deployment Template for Sagemaker Hosted C++ Scorer
+
+## Overview
+
+### Build Image
+
+Run the following command to build the docker image.
+
+```bash
+docker build -t <aws_account_id>.dkr.ecr.<region>.amazonaws.com/h2oai/sagemaker-hosted-scorer:<tag> .
+```
+
+Run the following command to build the docker image with gpu support.
+
+```bash
+docker build -t <aws_account_id>.dkr.ecr.<region>.amazonaws.com/h2oai/sagemaker-hosted-scorer:<tag> -f gpu.docker .
+```
+
+
+Verify that the Docker image was created, and take note of the version created.
+
+```bash
+docker images --format "{{.Repository}} \t {{.Tag}}" | grep "h2oai/sagemaker-hosted-scorer"
+```
+
+### Optional: Test the build
+
+After building, run to test the produced Docker container locally like this:
+
+Step 1:  Put a pipeline.mojo and valid license.sig into this directory (aws-sagemaker-hosted-scorer-cpp).
+
+Step 2:  Start the docker instance.
+
+
+```
+docker run \
+    --rm \
+    --init \
+    -ti \
+    -v `pwd`:/opt/ml/model \
+    -p 8080:8080 \
+    harbor.h2o.ai/opsh2oai/h2oai/sagemaker-hosted-scorer:<tag>
+```
+
+And to run the gpu image.
+
+```
+docker run \
+    --gpus all \
+    --rm \
+    --init \
+    -ti \
+    -v `pwd`:/opt/ml/model \
+    -p 8080:8080 \
+    harbor.h2o.ai/opsh2oai/h2oai/sagemaker-hosted-scorer:<tag>
+```
+
+(the number of web server workers can be configured by setting the environment variable: `WEB_SERVER_WORKERS` in the docker run command)
+
+Recommended parameters:
+* `OMP_NUM_THREADS=8` and can be increased for long size texts.
+* `cores/(2*OMP_NUM_THREADS) <= WEB_SERVER_WORKERS <= cores/OMP_NUM_THREADS`
+* `OMP_NUM_THREADS*WEB_SERVER_WORKERS` MUST NOT exceed `cores`
+
+Step 3:  Use the following curl command to test the container locally:
+
+```
+curl \
+    -X POST \
+    -H "Content-Type: application/json" \
+    -d @payload.json http://localhost:8080/invocations
+```
+
+payload.json:
+
+```
+{
+  "fields": [
+    "field1", "field2"
+  ],
+  "includeFieldsInOutput": [
+    "field2"
+  ],
+  "rows": [
+    [
+      "value1", "value2"
+    ],
+    [
+      "value1", "value2"
+    ]
+  ]
+}
+```
+
+
+### Deploy to SageMaker
+
+Create `h2oai/sagemaker-hosted-scorer` repository in Sagemaker for the scorer service image.
+
+Use the output of the command below to `aws ecr login`:
+
+```
+aws ecr get-login-password --region <region> | docker login --username AWS --password-stdin <aws_account_id>.dkr.ecr.<region>.amazonaws.com
+```
+
+Then push the scorer service image to AWS ECR (Elastic Container Registry):
+
+```
+docker push <aws_account_id>.dkr.ecr.<region>.amazonaws.com/h2oai/sagemaker-hosted-scorer:<tag>
+```
+
+Then create a model package with the pipeline file and the license key, and copy it to S3:
+
+```
+tar cvf mojo.tar pipeline.mojo license.sig
+gzip mojo.tar
+aws s3 cp mojo.tar.gz s3://<your-bucket>/
+```
+
+Next create the appropriate model and endpoint on Sagemaker.
+Check that the endpoint is available with `aws sagemaker list-endpoints`.

aws-sagemaker-hosted-scorer-cpp/gpu.docker

@@ -0,0 +1,22 @@
+FROM nvidia/cuda:11.1.1-base-ubuntu20.04
+
+RUN apt-get update && \
+    apt-get install -y libopenblas-dev python3.8 python3-pip && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3.8 0 && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements_gpu.txt /tmp/requirements.txt
+
+RUN python -m pip install pip==21.1 && pip install -r /tmp/requirements.txt
+
+ENV DRIVERLESS_AI_LICENSE_FILE='/opt/ml/model/license.sig'
+ENV MOJO_FILE_PATH='/opt/ml/model/pipeline.mojo'
+ENV WEB_SERVER_WORKERS=4
+
+RUN mkdir -p /opt/ml/code
+
+COPY py/scorer/mojo_cpp_scorer.py /opt/ml/code
+
+WORKDIR /opt/ml/code
+
+ENTRYPOINT gunicorn -w ${WEB_SERVER_WORKERS} -b 0.0.0.0:8080 mojo_cpp_scorer:app

aws-sagemaker-hosted-scorer-cpp/py/scorer/mojo_cpp_scorer.py

@@ -0,0 +1,148 @@
+import logging
+import os
+import threading
+import time
+
+import daimojo.model
+import datatable as dt
+from flask import Flask, request
+from flask_restful import Resource, Api
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = Flask(__name__)
+api = Api(app)
+
+
+class MojoPipeline(object):
+    _model = None
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        if MojoPipeline._instance is None:
+            with MojoPipeline._lock:
+                if MojoPipeline._instance is None:
+                    MojoPipeline._instance = super(MojoPipeline, cls).__new__(cls)
+                    MojoPipeline._instance.setup()
+        return MojoPipeline._instance
+
+    def setup(self):
+        self._set_omp_threads()
+        mojo_file_path = os.getenv('MOJO_FILE_PATH')
+        self._model = daimojo.model(mojo_file_path)
+
+    def get_id(self):
+        return self._model.uuid
+
+    def get_feature_names(self):
+        """Return feature names"""
+        return self._model.feature_names
+
+    def get_types(self):
+        """Get column types"""
+        types = {}
+        for index, value in enumerate(self._model.feature_names):
+            types[value] = self._model.feature_types[index]
+        return types
+
+    def get_missing_values(self):
+        """Return mojo missing values"""
+        return self._model.missing_values
+
+    def get_prediction(self, d_frame):
+        """Score and return predictions on a given dataset"""
+        self._set_omp_threads()
+        return self._model.predict(d_frame)
+
+    @staticmethod
+    def _set_omp_threads():
+        os.environ['OMP_NUM_THREADS'] = str(min(8, int(os.cpu_count())/2))
+
+
+class ScorerError(Exception):
+    """Base Scorer Error"""
+    status_code = 500
+
+
+class BadRequest(ScorerError):
+    """Bad request"""
+    status_code = 400
+
+
+def score(request_body, timeit=False):
+    if request_body is None or len(request_body.keys()) == 0:
+        raise BadRequest("Invalid request. Need a request body.")
+
+    if 'fields' not in request_body.keys() or not isinstance(request_body['fields'], list):
+        raise BadRequest("Cannot determine the request column fields")
+
+    if 'rows' not in request_body.keys() or not isinstance(request_body['rows'], list):
+        raise BadRequest("Cannot determine the request rows")
+
+    mojo = MojoPipeline()
+
+    # properly define types based on the request elements order
+    d_frame = dt.Frame(
+        [list(x) for x in list(zip(*request_body['rows']))],
+        names=list(mojo.get_types().keys()),
+        stypes=list(mojo.get_types().values())
+    )
+
+    start = time.time() if timeit else 0
+    result_frame = mojo.get_prediction(d_frame)
+    delta = time.time() - start if timeit else 0
+    request_id = mojo.get_id()
+
+    # check whether 'includeFieldsInOutput' comes with request body and combined them with scored result
+    if 'includeFieldsInOutput' in request_body:
+        include_fields = list(request_body['includeFieldsInOutput'])
+        if len(include_fields) > 0:
+            result_frame = get_combined_frame(d_frame, result_frame, include_fields)
+
+    ret = {
+        'id': request_id,
+        'fields': result_frame.names,
+        'score': result_frame.to_list()
+    }
+    if timeit:
+        ret['time'] = delta
+
+    return ret
+
+
+def get_combined_frame(input_frame, result_frame, include_on_out_fields):
+    combined_frame = dt.Frame()
+
+    for include_column in include_on_out_fields:
+        combined_frame = dt.cbind(combined_frame, input_frame[include_column])
+
+    combined_frame = dt.cbind(combined_frame, result_frame)
+    return combined_frame
+
+
+class ScorerAPI(Resource):
+    def post(self):
+        request_body = request.get_json()
+        try:
+            scoring_result = score(request_body, timeit='timeit' in request.args.keys())
+        except ScorerError as e:
+            return {'message': str(e)}, e.status_code
+        except Exception as exc:
+            return {'message': str(exc)}, 500
+        return scoring_result, 200
+
+
+class PingAPI(Resource):
+
+    def get(self):
+        pass
+
+
+api.add_resource(ScorerAPI, '/invocations')
+api.add_resource(PingAPI, '/ping')
+
+if __name__ == '__main__':
+    logger.info('==== Starting the H2O mojo-cpp scoring server =====')
+    app.run(host='0.0.0.0', port=8080, threaded=False)

aws-sagemaker-hosted-scorer-cpp/requirements.txt

@@ -0,0 +1,19 @@
+aniso8601==9.0.1
+click==8.1.3
+datatable==1.0.0
+Flask==2.1.2
+Flask-RESTful==0.3.9
+gunicorn==20.1.0
+importlib-metadata==4.11.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+numpy==1.22.4
+pandas==1.4.2
+protobuf==4.21.1
+python-dateutil==2.8.2
+pytz==2022.1
+six==1.16.0
+Werkzeug==2.1.2
+zipp==3.8.0
+https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/daimojo/2.7.9%2Bcu111.master.446/x86_64-centos7/daimojo-2.7.9%2Bmaster.446-cp38-cp38-linux_x86_64.whl

aws-sagemaker-hosted-scorer-cpp/requirements_gpu.txt

@@ -0,0 +1,19 @@
+aniso8601==9.0.1
+click==8.1.3
+datatable==1.0.0
+Flask==2.1.2
+Flask-RESTful==0.3.9
+gunicorn==20.1.0
+importlib-metadata==4.11.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+numpy==1.22.4
+pandas==1.4.2
+protobuf==4.21.1
+python-dateutil==2.8.2
+pytz==2022.1
+six==1.16.0
+Werkzeug==2.1.2
+zipp==3.8.0
+https://s3.amazonaws.com/artifacts.h2o.ai/releases/ai/h2o/daimojo/2.7.9%2Bmaster.450/x86_64-centos7/daimojo-2.7.9%2Bcu111.master.450-cp38-cp38-linux_x86_64.whl