Add project build upload (#14)

v3io · Mar 28, 2019 · 66a79c7 · 66a79c7
1 parent 3055c28
commit 66a79c7
Show file tree

Hide file tree

Showing 2 changed files with 100 additions and 15 deletions.
diff --git a/Makefile b/Makefile
@@ -1,22 +1,33 @@
+SRC_BINARY_NAME ?= "igz-fuse"
+DST_BINARY_NAME ?= "igz-fuse"
+FETCH_METHOD ?= "download"
+MIRROR ?=
+IGUAZIO_VERSION ?=
+
 RPM_PATH = "iguazio_yum"
 DEB_PATH = "iguazio_deb"
-BINARY_NAME = "igz-fuse"
-VERSION = $(IGUAZIO_VERSION)
 
 .PHONY: build
 build:
 	docker build --tag flex-fuse:unstable .
 
 .PHONY: download
 download:
-	@rm -rf hack/libs/${BINARY_NAME}*
-	@cd hack/libs && wget --quiet $(MIRROR)/$(RPM_PATH)/$(IGUAZIO_VERSION)/$(BINARY_NAME).rpm
-	@cd hack/libs && wget --quiet $(MIRROR)/$(DEB_PATH)/$(IGUAZIO_VERSION)/$(BINARY_NAME).deb
+	@rm -rf hack/libs/${DST_BINARY_NAME}*
+	@wget --quiet $(MIRROR)/$(RPM_PATH)/$(IGUAZIO_VERSION)/$(SRC_BINARY_NAME).rpm -O hack/libs/$(DST_BINARY_NAME).rpm
+	@wget --quiet $(MIRROR)/$(DEB_PATH)/$(IGUAZIO_VERSION)/$(SRC_BINARY_NAME).deb -O hack/libs/$(DST_BINARY_NAME).deb
+	@touch hack/libs/$(IGUAZIO_VERSION)
+
+.PHONY: copy
+copy:
+	@rm -rf hack/libs/${DST_BINARY_NAME}*
+	@cp $(MIRROR)/$(SRC_BINARY_NAME).rpm hack/libs/$(DST_BINARY_NAME).rpm
+	@cp $(MIRROR)/$(SRC_BINARY_NAME).deb hack/libs/$(DST_BINARY_NAME).deb
 	@touch hack/libs/$(IGUAZIO_VERSION)
 
 .PHONY: release
-release: check-req download build
-	docker tag flex-fuse:unstable flex-fuse:$(VERSION)
+release: check-req $(FETCH_METHOD) build
+	docker tag flex-fuse:unstable flex-fuse:$(IGUAZIO_VERSION)
 
 .PHONY: lint
 lint: ensure-gopath

diff --git a/zetup.py b/zetup.py
@@ -2,11 +2,13 @@
 
 import os
 import simplejson
+import tempfile
 
 from twisted.internet import defer
 
 import ziggy.utils
 import ziggy.fs
+import ziggy.docker
 import ziggy.tasks
 import ziggy.shell
 
@@ -75,20 +77,33 @@ def task_verify_zetup_unchanged(project):
 
 
 @defer.inlineCallbacks
-def task_build_images(project, version, mirror):
+def task_build_images(project, version, mirror=None, nas_deployed_artifacts_path='/mnt/nas'):
     """
     Internal build function
     """
 
-    project.logger.info('Building', version=version, mirror=mirror)
+    project.logger.info('Building',
+                        version=version,
+                        mirror=mirror,
+                        nas_deployed_artifacts_path=nas_deployed_artifacts_path)
 
     cwd = project.config['flex-fuse']['flex_fuse_path']
-    cmd = 'make release MIRROR={0} IGUAZIO_VERSION={1}'.format(mirror, version)
-
-    project.logger.debug('Building a release candidate', cwd=cwd, cmd=cmd)
-
-    out, _, _ = yield ziggy.shell.run(project.ctx, cmd, cwd=cwd)
-
+    cmd = 'make release'
+
+    env = os.environ.copy()
+    env['FETCH_METHOD'] = 'download'
+    env['MIRROR'] = mirror
+    env['IGUAZIO_VERSION'] = version
+    env['SRC_BINARY_NAME'] = 'igz-fuse'
+    env['DST_BINARY_NAME'] = 'igz-fuse'
+
+    if not mirror:
+        env['FETCH_METHOD'] = 'copy'
+        env['SRC_BINARY_NAME'] = 'fuse'
+        env['MIRROR'] = os.path.join(nas_deployed_artifacts_path, 'engine/zeek-packages')
+
+    project.logger.debug('Building a release candidate', cwd=cwd, cmd=cmd, env=env)
+    out, _, _ = yield ziggy.shell.run(project.ctx, 'make release', cwd=cwd, env=env)
     project.logger.info('Build images task is done', out=out)
 
 
@@ -145,6 +160,65 @@ def task_push_images(project, repository, tag, pushed_images_file_path):
     project.logger.info('Push images task is done', pushed_images=pushed_images)
 
 
+@defer.inlineCallbacks
+def task_project_build(project, output_dir, tag=None, nas_deployed_artifacts_path='/mnt/nas'):
+    project.ctx.info('Building', output_dir=output_dir, tag=tag)
+    tasks_to_run = [
+        {
+            'name': 'build_images',
+            'args': {
+                'version': tag,
+                'nas_deployed_artifacts_path': nas_deployed_artifacts_path,
+            },
+        },
+        {
+            'name': 'save_images',
+            'args': {
+                'output_filepath': os.path.join(output_dir, 'flex-fuse-docker-images.tar.gz'),
+                'images': ['flex-fuse:{}'.format(tag)],
+            },
+        },
+    ]
+
+    yield project.task_manager.run_tasks(project, tasks_to_run)
+    project.ctx.info('Finished building',
+                     output_dir=output_dir,
+                     tag=tag,
+                     nas_deployed_artifacts_path=nas_deployed_artifacts_path)
+
+
+@defer.inlineCallbacks
+def task_save_images(project, images, output_filepath=None):
+    if not output_filepath:
+        output_filepath = tempfile.mktemp(suffix='.tar.gz', prefix='flex-fuse-docker-')
+        project.logger.debug('no output filepath was given, using a temporary file',
+                             output_filepath=output_filepath)
+
+    project.logger.debug('Saving docker images', images=images)
+    yield ziggy.docker.save_images(project.ctx, images, output_filepath, compress=True)
+    project.logger.debug('Done saving docker images', output_filepath=output_filepath)
+
+
+@defer.inlineCallbacks
+def task_upload(project, upload_manifest_filepath, output_dir):
+    project.ctx.info('Uploading',
+                     upload_manifest_filepath=upload_manifest_filepath,
+                     output_dir=output_dir)
+
+    upload_links_content = ziggy.fs.read_file_contents(project.ctx, upload_manifest_filepath)
+    upload_links = simplejson.loads(upload_links_content)
+
+    # we will sync\upload out working dir to each links destination
+    for link in upload_links:
+        link['src'] = output_dir
+
+    yield ziggy.tasks.upload(project, links=upload_links)
+
+    project.ctx.info('Done uploading',
+                     upload_links=upload_links,
+                     output_dir=output_dir)
+
+
 @defer.inlineCallbacks
 def task_workflow(project, skipped_tasks=None):
     skipped_tasks = ziggy.utils.as_list(skipped_tasks) or []