-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #157 from jpolchlo/feature/add-railway-stats
Staging deployment and Railway statistics
- Loading branch information
Showing
32 changed files
with
870 additions
and
303 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1 @@ | ||
| repository | ||
| docker-compose.local.yml | ||
| docker-compose.deploy.yml | ||
| config-local.mk | ||
| config-aws.mk | ||
| config-*.mk |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,104 +1,106 @@ | ||
| include config-aws.mk # Variables for AWS options | ||
| include config-local.mk # Variables related to running locally | ||
| include config-deployment.mk | ||
|
|
||
| # The osmesa container | ||
| LOCAL_IMG := osm_analytics:latest | ||
| # If the user is on master branch, see if we should deploy to production | ||
| VERSION_TAG=$(shell ./scripts/get-tag.sh) | ||
| ifeq ($(VERSION_TAG), production) | ||
| DATABASE=${PRODUCTION_DB} | ||
| ECS_CLUSTER=${CLUSTER_NAME_DEPLOYMENT} | ||
| TASK_SUFFIX= | ||
| else | ||
| DATABASE=${STAGING_DB} | ||
| ECS_CLUSTER=${CLUSTER_NAME_STAGING} | ||
| TASK_SUFFIX=-staging | ||
| endif | ||
| DB_URI=${DB_BASE_URI}/${DATABASE} | ||
|
|
||
| .EXPORT_ALL_VARIABLES: | ||
|
|
||
| ######### | ||
| # LOCAL # | ||
| ######### | ||
| docker-compose.local.yml: | ||
| export LOCAL_IMG=${LOCAL_IMG}; \ | ||
| export AUGDIFF_SOURCE=${LOCAL_AUGDIFF_SOURCE}; \ | ||
| export CHANGESET_SOURCE=${LOCAL_CHANGESET_SOURCE}; \ | ||
| export CHANGE_SOURCE=${LOCAL_CHANGE_SOURCE}; \ | ||
| export AUGDIFF_START=${LOCAL_AUGDIFF_START}; \ | ||
| export CHANGESET_START=${LOCAL_CHANGESET_START}; \ | ||
| export CHANGE_START=${LOCAL_CHANGE_START}; \ | ||
| ./expand.sh docker-compose.local.yml.tpl > docker-compose.local.yml | ||
| ############################# | ||
| # Docker image management # | ||
| ############################# | ||
|
|
||
| start-local: docker-compose.local.yml | ||
| docker-compose -f docker-compose.local.yml up | ||
| .PHONY: build-container login-aws-registry tag-image push-image | ||
|
|
||
| stop-local: | ||
| docker-compose -f docker-compose.local.yml down | ||
|
|
||
|
|
||
| ######### | ||
| # AWS # | ||
| ######### | ||
| build-container: | ||
| cd .. && VERSION_TAG=${VERSION_TAG} ./build-container.sh | ||
|
|
||
| login-aws-registry: | ||
| eval `aws ecr get-login --no-include-email --region ${AWS_REGION}` | ||
|
|
||
| tag-image: | ||
| docker tag ${LOCAL_IMG} ${ECR_REPO} | ||
| tag-image: build-container | ||
| docker tag osm_analytics:${VERSION_TAG} ${ECR_IMAGE}:${VERSION_TAG} | ||
|
|
||
| push-image: login-aws-registry tag-image | ||
| docker push ${ECR_REPO} | ||
|
|
||
| .PHONY: docker-compose.deploy.yml | ||
|
|
||
| docker-compose.deploy.yml: docker-compose.deploy.yml.tpl | ||
| export ECR_REPO=${ECR_REPO} | ||
| export AWS_LOG_GROUP=${AWS_LOG_GROUP}; \ | ||
| export AWS_REGION=${AWS_REGION}; \ | ||
| export AUGDIFF_SOURCE=${AUGDIFF_SOURCE}; \ | ||
| export AUGDIFF_START=${AUGDIFF_START}; \ | ||
| export CHANGESET_SOURCE=${CHANGESET_SOURCE}; \ | ||
| export CHANGESET_START=${CHANGESET_START}; \ | ||
| export DB_URI=${DB_URI}; \ | ||
| ./expand.sh $< > $@ | ||
|
|
||
| .PHONY: configure-cluster | ||
|
|
||
| configure-cluster: | ||
| ecs-cli configure \ | ||
| --cluster ${CLUSTER_NAME} \ | ||
| --region ${AWS_REGION} \ | ||
| --config-name ${CONFIG_NAME} | ||
|
|
||
| cluster-up: | ||
| ecs-cli up \ | ||
| --keypair ${KEYPAIR} \ | ||
| --instance-role ${INSTANCE_ROLE} \ | ||
| --security-group ${SECURITY_GROUP} \ | ||
| --size 1 \ | ||
| --instance-type ${INSTANCE_TYPE} \ | ||
| --cluster-config ${CONFIG_NAME} \ | ||
| --subnets ${SUBNETS} \ | ||
| --vpc ${VPC} \ | ||
| --force \ | ||
| --verbose | ||
|
|
||
| cluster-down: | ||
| ecs-cli down --cluster-config ${CONFIG_NAME} | ||
|
|
||
| .PHONY: create-service | ||
|
|
||
| create-service: docker-compose.deploy.yml configure-cluster | ||
| ecs-cli compose \ | ||
| --file $< create \ | ||
| --cluster ${CLUSTER_NAME} | ||
|
|
||
| start-service: docker-compose.deploy.yml configure-cluster create-service | ||
| ecs-cli compose --file $< service up \ | ||
| --deployment-min-healthy-percent 0 \ | ||
| --create-log-groups \ | ||
| --cluster ${CLUSTER_NAME} | ||
|
|
||
| stop-service: docker-compose.deploy.yml | ||
| ecs-cli compose --file $< down | ||
|
|
||
|
|
||
| ######### | ||
| # ALL # | ||
| ######### | ||
| build-container: | ||
| cd .. && ./build-container.sh | ||
|
|
||
| clean: | ||
| rm -f docker-compose.local.yml | ||
| rm -f docker-compose.deploy.yml | ||
|
|
||
| docker push ${ECR_IMAGE}:${VERSION_TAG} | ||
|
|
||
| ####################### | ||
| # Streaming AWS Tasks # | ||
| ####################### | ||
|
|
||
| .PHONY: create-log-groups define-streaming-vectortile-tasks define-staging-streaming-update-tasks define-production-streaming-update-tasks deploy-streaming-footprint-updater deploy-streaming-edit-histogram-updater deploy-streaming-stats-updaters | ||
|
|
||
| create-log-groups: | ||
| ./scripts/create-log-groups.sh | ||
|
|
||
| define-streaming-vectortile-tasks: | ||
| ./scripts/define-streaming-vectortile-tasks.sh | ||
|
|
||
| define-staging-streaming-update-tasks: | ||
| ./scripts/define-staging-streaming-update-tasks.sh | ||
|
|
||
| define-production-streaming-update-tasks: | ||
| ./scripts/define-production-streaming-update-tasks.sh | ||
|
|
||
| stop-streaming-footprint-updater: | ||
| ./scripts/stop-streaming-service.sh streaming-user-footprint-tile-updater | ||
|
|
||
| deploy-streaming-footprint-updater: stop-streaming-footprint-updater | ||
| aws ecs create-service \ | ||
| --cluster "${CLUSTER_NAME_DEPLOYMENT}" \ | ||
| --service-name "streaming-user-footprint-tile-updater" \ | ||
| --task-definition "streaming-edit-histogram-tile-updater" \ | ||
| --desired-count 1 \ | ||
| --launch-type FARGATE \ | ||
| --scheduling-strategy REPLICA \ | ||
| --network-configuration ${NETWORK_CONFIGURATION} | ||
|
|
||
| stop-streaming-edit-histogram-updater: | ||
| ./scripts/stop-streaming-service.sh streaming-edit-histogram-tile-updater | ||
|
|
||
| deploy-streaming-edit-histogram-updater: stop-streaming-edit-histogram-updater | ||
| aws ecs create-service \ | ||
| --cluster "${CLUSTER_NAME_DEPLOYMENT}" \ | ||
| --service-name "streaming-edit-histogram-tile-updater" \ | ||
| --task-definition "streaming-edit-histogram-tile-updater" \ | ||
| --desired-count 1 \ | ||
| --launch-type FARGATE \ | ||
| --scheduling-strategy REPLICA \ | ||
| --network-configuration ${NETWORK_CONFIGURATION} | ||
|
|
||
| stop-streaming-stats-updaters: | ||
| ./scripts/stop-streaming-service.sh streaming-stats-updater | ||
|
|
||
| deploy-streaming-stats-updaters: stop-streaming-stats-updaters | ||
| aws ecs create-service \ | ||
| --cluster "${ECS_CLUSTER}" \ | ||
| --service-name "streaming-stats-updater" \ | ||
| --task-definition "streaming-stats-updater${TASK_SUFFIX}" \ | ||
| --desired-count 1 \ | ||
| --launch-type FARGATE \ | ||
| --scheduling-strategy REPLICA \ | ||
| --network-configuration ${NETWORK_CONFIGURATION} | ||
|
|
||
| deploy-streaming-vectortile-tasks: deploy-footprint-updater deploy-streaming-edit-histogram-updater | ||
|
|
||
| ################### | ||
| # Batch AWS Tasks # | ||
| ################### | ||
|
|
||
| batch-generate-footprints: | ||
| ./scripts/batch-generate-footprints.sh | ||
|
|
||
| batch-generate-edit-histograms: | ||
| ./scripts/batch-generate-edit-histograms.sh | ||
|
|
||
| batch-generate-db-backfill: | ||
| ./scripts/batch-process.sh "OSMesa Batch Process" "ChangesetStatsCreator" 64 "[\"spark-submit\", \"--deploy-mode\", \"cluster\", \"--class\", \"osmesa.analytics.oneoffs.ChangesetStatsCreator\", \"--conf\", \"spark.executor.memoryOverhead=2g\", \"--conf\", \"spark.sql.shuffle.partitions=2000\", \"--conf\", \"spark.speculation=true\", \"${OSMESA_ANALYTICS_JAR}\", \"--history\", \"${HISTORY_ORC}\", \"--changesets\", \"${CHANGESETS_ORC}\", \"--changeset-stream\", \"${CHANGESET_SOURCE}\", \"--database-url\", \"${DB_URI}\"]" |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| ################################################################################ | ||
| # AWS properties | ||
| ################################################################################ | ||
| export KEYPAIR := | ||
| export SUBNET := | ||
| export AWS_REGION := us-east-1 | ||
| export IAM_ACCOUNT := | ||
|
|
||
| ################################################################################ | ||
| # Streaming resource definitions | ||
| ################################################################################ | ||
| export CLUSTER_NAME := osm-stat-stream-cluster | ||
| export STREAMING_INSTANCE_TYPE := m4.xlarge | ||
| export ECR_IMAGE := | ||
| export AWS_LOG_GROUP := streaming-stats-updater | ||
| export ECS_SUBNET := ${SUBNET} | ||
| export ECS_SECURITY_GROUP := | ||
|
|
||
| export AUGDIFF_SOURCE := | ||
| export CHANGESET_SOURCE := | ||
|
|
||
| export DB_BASE_URI := | ||
| export PRODUCTION_DB := | ||
| export STAGING_DB := | ||
|
|
||
| export NETWORK_CONFIGURATION="{\"awsvpcConfiguration\": {\"subnets\": [\"${ECS_SUBNET}\"], \"securityGroups\": [\"${ECS_SECURITY_GROUP}\"], \"assignPublicIp\": \"DISABLED\"}}" | ||
|
|
||
| ################################################################################ | ||
| # Batch resource definitions | ||
| ################################################################################ | ||
| export SERVICE_ACCESS_SECURITY_GROUP := ${ECS_SECURITY_GROUP} | ||
| export EMR_MASTER_SECURITY_GROUP := | ||
| export EMR_SLAVE_SECURITY_GROUP := | ||
|
|
||
| export BATCH_INSTANCE_TYPE := m4.xlarge | ||
| export OSMESA_ANALYTICS_JAR := s3://<bucket>/osmesa-analytics.jar | ||
|
|
||
| export HISTORY_ORC := | ||
| export CHANGESETS_ORC := | ||
|
|
||
| export FOOTPRINT_VT_LOCATION := | ||
| export HISTOGRAM_VT_LOCATION := |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.