Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

commands: Launch external image ingestion in the CLI to Preview. #426

Merged
merged 1 commit into from
Nov 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions python/ee/cli/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -1448,6 +1448,43 @@ def is_tf_record(path: str) -> bool:
return manifest


class UploadExternalImageCommand:
"""Creates an asset backed by an external image.

See
https://developers.google.com/earth-engine/Earth_Engine_asset_from_cloud_geotiff
for more details on constructing the manifest.
"""

name = 'external_image'

def __init__(self, parser: argparse.ArgumentParser):
_add_wait_arg(parser)
_add_overwrite_arg(parser)
parser.add_argument(
'--manifest',
help='Local path to a JSON asset manifest file.')

@_using_v1alpha
def run(
self, args: argparse.Namespace, config: utils.CommandLineConfig
) -> None:
"""Creates an external image synchronously."""
config.ee_init()
manifest = self.manifest_from_args(args)
name = ee.data.startExternalImageIngestion(manifest, args.force)['name']
print('Created asset %s' % name)

def manifest_from_args(self, args: argparse.Namespace) -> Dict[str, Any]:
"""Constructs an upload manifest from the command-line flags."""

if args.manifest:
with open(args.manifest) as fh:
return json.loads(fh.read())

raise ValueError('Flag --manifest must be set.')


# TODO(user): update src_files help string when secondary files
# can be uploaded.
class UploadTableCommand:
Expand Down Expand Up @@ -1640,6 +1677,17 @@ class UploadCommand(Dispatcher):
UploadTableCommand,
]


class AlphaUploadCommand(Dispatcher):
"""Uploads assets to Earth Engine."""

name = 'upload'

COMMANDS = [
UploadExternalImageCommand,
]


class _UploadManifestBase:
"""Uploads an asset to Earth Engine using the given manifest file."""

Expand Down Expand Up @@ -2016,6 +2064,7 @@ class AlphaCommand(Dispatcher):

COMMANDS = [
ProjectConfigCommand,
AlphaUploadCommand,
]


Expand Down
37 changes: 37 additions & 0 deletions python/ee/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -1910,6 +1910,7 @@ def _prepare_and_run_export(

# TODO(user): use StrEnum when 3.11 is the min version
_INTERNAL_IMPORT = 'INTERNAL_IMPORT'
_EXTERNAL_IMPORT = 'EXTERNAL_IMPORT'


def _startIngestion(
Expand All @@ -1933,6 +1934,10 @@ def _startIngestion(
image = _get_cloud_projects().image()
if import_mode == _INTERNAL_IMPORT:
import_request = image.import_(project=_get_projects_path(), body=request)
elif import_mode == _EXTERNAL_IMPORT:
import_request = image.importExternal(
project=_get_projects_path(), body=request
)
else:
raise ee_exception.EEException(
'{} is not a valid import mode'.format(import_mode)
Expand Down Expand Up @@ -1993,6 +1998,38 @@ def startIngestion(
return _startIngestion(request_id, params, allow_overwrite, _INTERNAL_IMPORT)


def startExternalImageIngestion(
image_manifest: Dict[str, Any],
allow_overwrite: bool = False,
) -> Dict[str, Any]:
"""Creates an external image.

Args:
image_manifest: The object that describes the import task, which can
have these fields:
name (string) The destination asset id (e.g.,
"projects/myproject/assets/foo/bar").
tilesets (array) A list of Google Cloud Storage source file paths
formatted like:
[{'sources': [
{'uris': ['foo.tif', 'foo.prj']},
{'uris': ['bar.tif', 'bar.prj']},
]}]
Where path values correspond to source files' Google Cloud Storage
object names, e.g., 'gs://bucketname/filename.tif'
bands (array) An optional list of band names formatted like:
[{'id': 'R'}, {'id': 'G'}, {'id': 'B'}]
In general, this is a dict representation of an ImageManifest.
allow_overwrite: Whether the ingested image can overwrite an
existing version.

Returns:
The name of the created asset.
"""
return _startIngestion(
'unused', image_manifest, allow_overwrite, _EXTERNAL_IMPORT)


def startTableIngestion(
request_id: str, params: Dict[str, Any], allow_overwrite: bool = False
) -> Dict[str, Any]:
Expand Down
Loading