diff --git a/landcover_viewer/core.py b/landcover_viewer/core.py index 091373f..f166263 100644 --- a/landcover_viewer/core.py +++ b/landcover_viewer/core.py @@ -5,7 +5,7 @@ import ee, json, os, time -# ----------------------------------------------------------------------------- +# ============================================================================= class LandCoverViewer(): ''' Google Earth Engine API @@ -652,8 +652,7 @@ def download_to_drive(self, try: task = ee.batch.Export.image.toDrive( image = image, - folder = 'GEEServiceAccountExport', - description = 'Export from SERVIR Mekong Team', + description = 'Land Cover Export from SERVIR Mekong', fileNamePrefix = temp_file_name, scale = 30, region = self.geometry.bounds().getInfo()['coordinates'],#self.geometry.getInfo()['coordinates'], @@ -711,3 +710,5 @@ def get_stats(self, year=2016, primitives=range(0, 21)): # area = reducer.getInfo()['tcc'] * 100 * 100 * 0.0001 # in hectare # meaning we can use the value directly as the hectare return {self.INDEX_CLASS[int(float(k))]:float('{0:.2f}'.format(v)) for k,v in data.items()} + +# ============================================================================= diff --git a/requirements.txt b/requirements.txt index d65a4b8..a425180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,11 +31,12 @@ djangocms-style==2.0.2 djangocms-text-ckeditor==3.5.0 djangocms-video==2.0.3 djangorestframework==3.6.4 -earthengine-api==0.1.119 +earthengine-api==0.1.182 easy-thumbnails==2.4.1 enum34==1.1.6 fasteners==0.14.1 google-api-python-client==1.6.3 +google-auth==1.6.3 gunicorn==19.7.1 html5lib==0.9999999 httplib2==0.10.3 @@ -48,8 +49,8 @@ oauth2client==4.1.2 olefile==0.44 Pillow==4.2.1 psycopg2==2.7.3.1 -pyasn1==0.3.4 -pyasn1-modules==0.1.4 +pyasn1==0.4.5 +pyasn1-modules==0.2.5 pycparser==2.18 pyOpenSSL==17.2.0 pytz==2017.2 diff --git a/scripts/delete-service-account-files.py b/scripts/delete-service-account-files.py new file mode 100644 index 0000000..c8f04dc --- /dev/null +++ b/scripts/delete-service-account-files.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +import os +import googleapiclient.discovery +from google.oauth2 import service_account + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# GEE authentication +# The service account email address authorized by your Google contact. +EE_ACCOUNT = 'rlcms-526@servir-rlcms.iam.gserviceaccount.com' +EE_PRIVATE_KEY_FILE = os.path.join(BASE_DIR, 'credentials/privatekey.json') + +GOOGLE_OAUTH2_CREDENTIALS = service_account.Credentials.from_service_account_file( + EE_PRIVATE_KEY_FILE, + scopes = ['https://www.googleapis.com/auth/drive'] +) + +drive_service = googleapiclient.discovery.build('drive', 'v2', credentials=GOOGLE_OAUTH2_CREDENTIALS) + +# list all the files +files = drive_service.files().list().execute() + +# get all the items of the files +items = files.get('items') +print('total {} items found'.format(len(items))) + +# delete the files by getting its id +deleted = 0 +not_deleted = 0 +not_deleted_id = [] + +for f in items: + response = drive_service.files().delete(fileId=f['id']).execute() + if not response: + deleted += 1 + else: + not_deleted_id.append(f['id']) + not_deleted += 1 + +print('total {} items deleted'.format(deleted)) +print('total {} items cannot be deleted having ids: {}'.format(not_deleted, not_deleted_id)) + +folders = drive_service.children().list(folderId='root').execute() +folders = folders.get('items') +print('total {} folder found'.format(len(folders))) + +for f in folders: + drive_service.children().delete(folderId='root', childId=f['id']).execute() diff --git a/utils/drive.py b/utils/drive.py index 173f0fd..12a5ca9 100644 --- a/utils/drive.py +++ b/utils/drive.py @@ -8,22 +8,26 @@ import googleapiclient.discovery import httplib2 -# ----------------------------------------------------------------------------- +# ============================================================================= class DriveHelper(object): """A helper class for interfacing with Google Drive.""" # ------------------------------------------------------------------------- - def __init__(self, credentials): + def __init__(self, credentials, use_http=False): """Creates a credentialed Drive service with the given credentials. Args: credentials: The OAuth2 credentials. """ - http = credentials.authorize(httplib2.Http()) - self.service = googleapiclient.discovery.build('drive', 'v2', http=http) + + if use_http: + http = credentials.authorize(httplib2.Http()) + self.service = googleapiclient.discovery.build('drive', 'v2', http=http) + else: + self.service = googleapiclient.discovery.build('drive', 'v2', credentials=credentials) # ------------------------------------------------------------------------- - def GrantAccess(self, file_id, email): + def grant_access(self, file_id, email): """Grants the email address write access to the file with the given ID. Note: "writer" access is required to create a copy of a file. @@ -32,6 +36,7 @@ def GrantAccess(self, file_id, email): file_id: The ID of the file to which to grant access. email: The email address to grant access. """ + # Determine the permission ID for the email address. id_resp = self.service.permissions().getIdForEmail(email=email).execute() new_permission = { @@ -40,12 +45,13 @@ def GrantAccess(self, file_id, email): 'id': id_resp['id'] } self.service.permissions().insert( - fileId=file_id, - body=new_permission, - sendNotificationEmails=False).execute() + fileId = file_id, + body = new_permission, + sendNotificationEmails = False + ).execute() # ------------------------------------------------------------------------- - def CopyFile(self, origin_file_id, copy_title): + def copy_file(self, origin_file_id, copy_title): """Copies the file with the ID in the user's root folder with the title. Args: @@ -55,13 +61,16 @@ def CopyFile(self, origin_file_id, copy_title): Returns: The file ID of the copy. """ + body = {'title': copy_title} copied_file = self.service.files().copy( - fileId=origin_file_id, body=body).execute() + fileId = origin_file_id, + body = body + ).execute() return copied_file['id'] # ------------------------------------------------------------------------- - def GetExportedFiles(self, prefix): + def get_exported_files(self, prefix): """Returns the Drive file ID of the first file found with the prefix. Args: @@ -70,16 +79,24 @@ def GetExportedFiles(self, prefix): Returns: A list of Drive file objects that match the prefix. """ + prefix_with_escaped_quotes = prefix.replace('"', '\\"') query = "title contains '%s'" % prefix_with_escaped_quotes - files = self.service.files().list(q=query).execute() + files = self.service.files().list( + q = query + ).execute() return files.get('items') # ------------------------------------------------------------------------- - def DeleteFile(self, file_id): + def delete_file(self, file_id): """Deletes the file with the given ID. Args: file_id: The ID of the file to delete. """ - self.service.files().delete(fileId=file_id).execute() \ No newline at end of file + + self.service.files().delete( + fileId = file_id + ).execute() + +# ============================================================================= diff --git a/utils/utils.py b/utils/utils.py index 4404c93..634625b 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -11,35 +11,38 @@ def get_unique_string(): """Returns a likely-to-be unique string.""" random_str = ''.join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) + random.choice(string.ascii_uppercase + string.digits) for _ in range(6) + ) date_str = str(int(time.time())) return date_str + random_str # ----------------------------------------------------------------------------- def transfer_files_to_user_drive(temp_file_name, user_email, user_id, file_name, oauth2object): - APP_DRIVE_HELPER = drive.DriveHelper(settings.GOOGLE_OAUTH2_CREDENTIALS) - files = APP_DRIVE_HELPER.GetExportedFiles(temp_file_name) + app_drive = drive.DriveHelper(settings.GOOGLE_OAUTH2_CREDENTIALS) + files = app_drive.get_exported_files(temp_file_name) # Grant the user write access to the file(s) in the app service account's Drive. for f in files: - APP_DRIVE_HELPER.GrantAccess(f['id'], user_email) + app_drive.grant_access(f['id'], user_email) - user_drive_helper = drive.DriveHelper(oauth2object) + user_drive = drive.DriveHelper(oauth2object, True) # Copy the file(s) into the user's Drive. if len(files) == 1: file_id = files[0]['id'] - copied_file_id = user_drive_helper.CopyFile(file_id, file_name) + copied_file_id = user_drive.copy_file(file_id, file_name) trailer = 'open?id=' + copied_file_id else: trailer = '' for f in files: # The titles of the files include the coordinates separated by a dash. coords = '-'.join(f['title'].split('-')[-2:]) - user_drive_helper.CopyFile(f['id'], file_name + '-' + coords) + user_drive.copy_file(f['id'], file_name + '-' + coords) # Delete the file from the service account's Drive. for f in files: - APP_DRIVE_HELPER.DeleteFile(f['id']) + app_drive.delete_file(f['id']) return 'https://drive.google.com/' + trailer + +# -----------------------------------------------------------------------------