Add commands for building and pushing frontend to github and building…

… and pushing the docker-image to ECR

README.md

@@ -192,6 +192,24 @@ DJANGO_DB=sqlite DJANGO_SETTINGS_MODULE=khan.settings.prod pytest -vv
 DJANGO_DB=default DJANGO_SETTINGS_MODULE=khan.settings.prod pytest -vv
 ```
 
+## Ping Data Intelligence Infra Scripts
+
+Frontend - 
+Only build = `python infra/build_frontend.py`
+Also commit and push to remote repo = `python infra/build_frontend.py -p`
+
+Backend - 
+Only build = `python infra/build_image.py` 
+Also push to ECR after build = 
+```bash
+# Assume pingintel-ml role using Granted
+assume
+
+# Create and push the image
+python infra/build_image.py -p
+```
+
+For more info see [the README for Infra Scripts](./infra/README.md)
 
 ## What you get from Label Studio
 

infra/README.md

@@ -0,0 +1,33 @@
+# Ping Data Intelligence Infra Scripts
+
+## Introduction
+
+Welcome to the Ping Data Intelligence Label Studio fork.
+This folder has all the required scripts to build, push and commit frontend changes and create a docker image to be pushed to ECR.
+
+## build_frontend
+
+To build the frontend you can run `python infra/build_frontend.py` in the root folder. This runs the yarn run build command to create frontend static files.
+
+This command can also be used to commit and push the changes in the develop repo so it recreates the static files for the frontend. 
+
+For this, run the command with `python infra/build_frontend.py -p` which pushes to the remote repository develop branch with a new commit with the message `Build Frontend from commit:latest_commit_hash` using the latest commit hash of the develop branch.
+
+This command can be further configured with two more options.
+
+`--message` : Changes the github commit message if you are using the command to push to github. Defaults to `Build Frontend from commit:latest_commit_hash'` with the latest commit hash in the develop branch.
+
+`--branch` : Changes the current branch and the branch this commit should be pushed to. This command switches to the given branch, builds the frontend and pushes to the same branch if this option is used. Defaults to `develop`
+
+## build_image
+
+Used to build a docker image and optionally push it to ECR. To create a docker image just run `python infra/build_image.py`.
+
+To push it to ECR you can supply it with the optional parameter `--push` or `-p`. This requires you to assume the role `pingintel-ml` using Granted. After assuming the correct role you can run `python infra/build_image.py -p`
+
+Script is configured to push to Ping Intelligence repository by default. But this can be configured using the following options when running the command.
+
+`--aws-acount`: Controls which account the image should be pushed to. 
+`--ecr-repo`: Configures which ECR repo it should push to. Defaults to `label-studio`
+`--aws-region`: Configures which AWS region the command should push. Defaults to `us-east-1`
+`--docker-image-tag`: Configures the name of the docker-image. Defaults to `pingintel/label-studio`

infra/build_frontend.py

@@ -0,0 +1,61 @@
+import os
+import subprocess
+import click
+import sys
+
+from infra.shared import log_message, execute
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+IS_WINDOWS = sys.platform == 'win32'
+
+if IS_WINDOWS:
+    def quote(s):
+        return s
+
+def build_frontend(message, branch, push):
+    """Build the frontend"""
+    log_message("Starting Frontend build")
+
+    log_message("Switching branches")
+    cmd = f"git switch {branch} || git switch -c {branch}"
+    execute(cmd)
+
+    web_dir = os.path.join(SCRIPT_DIR, "../web")
+    abs_dir = os.path.abspath(web_dir)
+    log_message(f"Changing directory to {abs_dir}")
+    os.chdir(abs_dir)
+    log_message("Running 'yarn run build'")
+    execute("yarn run build")
+
+    if push:
+        log_message("*** Committing and Pushing to Github ***")
+
+        cmd = f"git rev-parse HEAD"
+        latest_commit = subprocess.check_output(cmd, shell=True, text=True).strip()
+
+        cmd = f"git add ."
+        execute(cmd)
+
+        commit_message = f"Build Frontend from commit: {latest_commit}"
+
+        if message is not "default":
+            commit_message = message
+
+        cmd = f'git commit -m "{commit_message}"'
+        execute(cmd)
+
+        cmd = f"git push origin {branch}"
+        execute(cmd)
+
+    log_message("Done!")
+
+
+@click.command()
+@click.option("--message", help="Github commit message. Defaults to 'Build Frontend from commit:latest_commit_hash'", default="default")
+@click.option("--branch", help="Github branch to push to. Defaults to 'develop'", default="develop")
+@click.option("--push", "-p", help="Flag to enable github commit and push, defaults to False", is_flag=True, default=False)
+def run(message, branch ,push):
+    build_frontend(message, branch, push)
+
+if __name__ == "__main__":
+    run()

infra/build_image.py

@@ -0,0 +1,55 @@
+import os
+import click
+import sys
+
+from infra.shared import log_message, execute
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+IS_WINDOWS = sys.platform == 'win32'
+
+if IS_WINDOWS:
+    def quote(s):
+        return s
+
+def build_docker_image(aws_account, aws_region, ecr_repo, push, docker_image_tag):
+    """Build the Docker image"""
+
+    log_message("*** Starting Docker Image build process ***")
+
+    web_dir = os.path.join(SCRIPT_DIR, "../")
+    abs_dir = os.path.abspath(web_dir)
+    log_message(F"Changing directory to {abs_dir}")
+
+    os.chdir(abs_dir)
+    log_message("Running 'docker build'")
+    cmd = f"docker build -t {docker_image_tag} ."
+    execute(cmd)
+
+    if push:
+        log_message("*** Pushing to AWS ECR ***")
+
+        log_message("Logging in to ECR")
+        cmd = f"docker login -u AWS -p $(aws ecr get-login-password --region {aws_region}) {aws_account}.dkr.ecr.{aws_region}.amazonaws.com"
+        execute(cmd)
+
+        log_message("Tagging the image")
+        cmd = f"docker tag {docker_image_tag}:latest {aws_account}.dkr.ecr.{aws_region}.amazonaws.com/{ecr_repo}:latest"
+        execute(cmd)
+
+        log_message("Pushing")
+        cmd = f"docker push {aws_account}.dkr.ecr.{aws_region}.amazonaws.com/{ecr_repo}:latest"
+        execute(cmd)
+
+        log_message("Done!")
+
+@click.command()
+@click.option("--push", "-p", help="Flag to enable ECR push, defaults to False", is_flag=True, default=False)
+@click.option("--docker-image-tag", default="pingintel/label-studio")
+@click.option("--aws-account", default="654654387973")
+@click.option("--ecr-repo", default="label-studio")
+@click.option("--aws-region", help="Region, defaults to us-east-1", default="us-east-1")
+def run(push, aws_account, aws_region, ecr_repo, docker_image_tag):
+     build_docker_image(aws_account, aws_region, ecr_repo, push, docker_image_tag)
+
+if __name__ == "__main__":
+    run()

infra/shared.py

@@ -0,0 +1,46 @@
+import datetime
+import click
+import subprocess
+from timeit import default_timer as timer
+
+start_time = timer()
+
+def log_message(msg):
+    timestamp = datetime.datetime.utcnow().strftime(format="%Y/%m/%d %H:%M:%S")
+    elapsed = timer() - start_time
+    click.echo(f"{timestamp} {elapsed:.1f}s: {msg}")
+
+def execute(cmd):
+    """Execute a shell command and print the output"""
+    errorlevel = 0
+    process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+
+    try:
+        output = []
+        for line in process.stdout:
+            click.echo(line, nl=False)
+            output.append(line)
+        process.wait()
+
+        if process.returncode != 0:
+            raise subprocess.CalledProcessError(process.returncode, cmd, output="".join(output))
+    except KeyboardInterrupt:
+        process.terminate()
+        click.echo("\nAborted!")
+    except subprocess.CalledProcessError as exc:
+        errorlevel = exc.returncode
+        error_message = exc.output
+        if "Cannot connect to the Docker daemon" in error_message:
+            click.echo("[ERROR]: Cannot connect to the Docker daemon.")
+            click.echo("[HINT]: Make sure Docker is running.")
+        elif "Unable to locate credentials" in error_message:
+            click.echo("[ERROR]: Unable to locate credentials.")
+            click.echo("[HINT]: Make sure you assume the 'pingintel-ml' role using Granted.")
+        elif "denied:" in error_message:
+            click.echo("[ERROR]: Permission Denied")
+            click.echo("[HINT]: Make sure you assume the role 'pingintel-ml' using Granted. Your current role doesn't have the permissions required.")
+        else:
+            click.echo(f"Command failed (err {errorlevel}): {cmd}\n{error_message}")
+
+    if errorlevel != 0:
+        click.echo("Command failed (err %d): %s" % (errorlevel, cmd))