diff --git a/BlobHelper/BlobHelper.csproj b/BlobHelper/BlobHelper.csproj index 69583a6..cd7589e 100644 --- a/BlobHelper/BlobHelper.csproj +++ b/BlobHelper/BlobHelper.csproj @@ -3,7 +3,7 @@ netstandard2.0;net462 true - 1.2.2 + 1.3.0 Joel Christner BLOB storage wrapper for Microsoft Azure, Amazon S3, Kvpbase, and local filesystem written in C#. (c)2019 Joel Christner @@ -11,19 +11,19 @@ https://github.com/jchristn/BlobHelper Github https://github.com/jchristn/BlobHelper/blob/master/LICENSE.TXT - Add missing regions + Enumeration and object metadata. https://raw.githubusercontent.com/jchristn/BlobHelper/master/assets/icon.ico blob azure storage s3 object rest - + - - + + - + diff --git a/BlobHelper/BlobMetadata.cs b/BlobHelper/BlobMetadata.cs new file mode 100644 index 0000000..80a6529 --- /dev/null +++ b/BlobHelper/BlobMetadata.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace BlobHelper +{ + /// + /// Metadata about a BLOB. + /// + public class BlobMetadata + { + #region Public-Members + + /// + /// Object key. + /// + public string Key { get; set; } + + /// + /// Content type for the object. + /// + public string ContentType { get; set; } + + /// + /// Content length of the object. + /// + public long ContentLength { get; set; } + + /// + /// ETag of the object. + /// + public string ETag { get; set; } + + /// + /// Timestamp from when the object was created. + /// + public DateTime Created { get; set; } + + #endregion + + #region Private-Members + + #endregion + + #region Constructors-and-Factories + + /// + /// Instantiate the object. + /// + public BlobMetadata() + { + + } + + #endregion + + #region Public-Methods + + /// + /// Create a human-readable string of the object. + /// + /// String. + public override string ToString() + { + string ret = + "---" + Environment.NewLine + + " Key : " + Key + Environment.NewLine + + " Content Type : " + ContentType + Environment.NewLine + + " Content Length : " + ContentLength + Environment.NewLine + + " ETag : " + ETag + Environment.NewLine + + " Created : " + Created.ToString("yyyy-MM-dd HH:mm:ss") + Environment.NewLine; + + return ret; + } + + #endregion + + #region Private-Methods + + #endregion + } +} diff --git a/BlobHelper/Blobs.cs b/BlobHelper/Blobs.cs index 1621966..44a0e85 100644 --- a/BlobHelper/Blobs.cs +++ b/BlobHelper/Blobs.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Data; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; using KvpbaseSDK; @@ -41,7 +43,9 @@ public class Blobs private CloudBlobClient _AzureBlobClient; private CloudBlobContainer _AzureContainer; - private KvpbaseClient _Kvpbase; + private KvpbaseClient _Kvpbase; + + private ConcurrentDictionary _AzureContinuationTokens = new ConcurrentDictionary(); #endregion @@ -105,29 +109,29 @@ public Blobs(KvpbaseSettings config) #region Public-Methods /// - /// Delete a BLOB by its ID. + /// Delete a BLOB by its key. /// - /// ID of the BLOB. + /// Key of the BLOB. /// True if successful. - public async Task Delete(string id) + public async Task Delete(string key) { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); bool success = false; switch (_StorageType) { case StorageType.AwsS3: - success = await S3Delete(id); + success = await S3Delete(key); break; case StorageType.Azure: - success = await AzureDelete(id); + success = await AzureDelete(key); break; case StorageType.Disk: - success = await DiskDelete(id); + success = await DiskDelete(key); break; case StorageType.Kvpbase: - success = await KvpbaseDelete(id); + success = await KvpbaseDelete(key); break; default: throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); @@ -137,74 +141,191 @@ public async Task Delete(string id) } /// - /// Retrieve a BLOB by its ID. + /// Retrieve a BLOB. /// - /// ID of the BLOB. + /// Key of the BLOB. /// Byte array containing BLOB data. /// Byte data of the BLOB. - public async Task Get(string id) + public async Task Get(string key) { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); switch (_StorageType) { case StorageType.AwsS3: - return await S3Get(id); + return await S3Get(key); case StorageType.Azure: - return await AzureGet(id); + return await AzureGet(key); case StorageType.Disk: - return await DiskGet(id); + return await DiskGet(key); case StorageType.Kvpbase: - return await KvpbaseGet(id); + return await KvpbaseGet(key); default: throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); } } /// - /// Write a BLOB. + /// Retrieve a BLOB. + /// + /// Key of the BLOB. + /// Content length. + /// Stream. + /// True if successful. + public bool Get(string key, out long contentLength, out Stream stream) + { + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); + + switch (_StorageType) + { + case StorageType.AwsS3: + return S3Get(key, out contentLength, out stream); + case StorageType.Azure: + return AzureGet(key, out contentLength, out stream); + case StorageType.Disk: + return DiskGet(key, out contentLength, out stream); + case StorageType.Kvpbase: + return KvpbaseGet(key, out contentLength, out stream); + default: + throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); + } + } + + /// + /// Write a BLOB using a byte array. /// - /// ID of the BLOB. + /// Key of the BLOB. /// True of the supplied data is a string containing Base64-encoded data. /// BLOB data. /// True if successful. public async Task Write( - string id, + string key, string contentType, byte[] data) { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); if (_StorageType != StorageType.Disk && String.IsNullOrEmpty(contentType)) throw new ArgumentNullException(nameof(contentType)); switch (_StorageType) { case StorageType.AwsS3: - return await S3Write(id, contentType, data); + return await S3Write(key, contentType, data); case StorageType.Azure: - return await AzureWrite(id, contentType, data); + return await AzureWrite(key, contentType, data); case StorageType.Disk: - return await DiskWrite(id, data); + return await DiskWrite(key, data); case StorageType.Kvpbase: - return _Kvpbase.WriteObject(_KvpbaseSettings.Container, id, contentType, data); + return await KvpbaseWrite(key, contentType, data); default: throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); } } - public async Task Exists(string id) + /// + /// Write a BLOB. + /// + /// Key of the BLOB. + /// Content type. + /// Content length. + /// Stream containing the data. + /// True if successful. + public bool Write( + string key, + string contentType, + long contentLength, + Stream stream) + { + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); + if (contentLength < 0) throw new ArgumentException("Content length must be zero or greater."); + if (stream == null) throw new ArgumentNullException(nameof(stream)); + if (!stream.CanRead) throw new IOException("Cannot read from supplied stream."); + + switch (_StorageType) + { + case StorageType.AwsS3: + return S3Write(key, contentType, contentLength, stream); + case StorageType.Azure: + return AzureWrite(key, contentType, contentLength, stream); + case StorageType.Disk: + return DiskWrite(key, contentLength, stream); + case StorageType.Kvpbase: + return KvpbaseWrite(key, contentType, contentLength, stream); + default: + throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); + } + } + + /// + /// Check if a BLOB exists. + /// + /// Key of the BLOB. + /// True if exists. + public async Task Exists(string key) { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); switch (_StorageType) { case StorageType.AwsS3: - return await S3Exists(id); + return await S3Exists(key); case StorageType.Azure: - return await AzureExists(id); + return await AzureExists(key); case StorageType.Disk: - return await DiskExists(id); + return await DiskExists(key); case StorageType.Kvpbase: - return await KvpbaseExists(id); + return await KvpbaseExists(key); + default: + throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); + } + } + + /// + /// Retrieve BLOB metadata. + /// + /// Key of the BLOB. + /// Metadata. + /// True if successful. + public bool GetMetadata(string key, out BlobMetadata md) + { + if (String.IsNullOrEmpty(key)) throw new ArgumentNullException(nameof(key)); + + switch (_StorageType) + { + case StorageType.AwsS3: + return S3GetMetadata(key, out md); + case StorageType.Azure: + return AzureGetMetadata(key, out md); + case StorageType.Disk: + return DiskGetMetadata(key, out md); + case StorageType.Kvpbase: + return KvpbaseGetMetadata(key, out md); + default: + throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); + } + } + + /// + /// Enumerate BLOBs. + /// + /// Key prefix that must match. + /// Continuation token to use for subsequent enumeration requests. + /// Next continuation token to supply should you want to continue enumerating from the end of the current response. + /// List of BLOB metadata. + /// True if successful. + public bool Enumerate(string continuationToken, out string nextContinuationToken, out List blobs) + { + blobs = new List(); + + switch (_StorageType) + { + case StorageType.AwsS3: + return S3Enumerate(continuationToken, out nextContinuationToken, out blobs); + case StorageType.Azure: + return AzureEnumerate(continuationToken, out nextContinuationToken, out blobs); + case StorageType.Disk: + return DiskEnumerate(continuationToken, out nextContinuationToken, out blobs); + case StorageType.Kvpbase: + return KvpbaseEnumerate(continuationToken, out nextContinuationToken, out blobs); default: throw new ArgumentException("Unknown storage type: " + _StorageType.ToString()); } @@ -260,33 +381,126 @@ private void InitializeClients() #region Private-Kvpbase-Methods #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task KvpbaseDelete(string id) + private async Task KvpbaseDelete(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - return _Kvpbase.DeleteObject(_KvpbaseSettings.Container, id); + return _Kvpbase.DeleteObject(_KvpbaseSettings.Container, key); } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task KvpbaseGet(string id) + private async Task KvpbaseGet(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - byte[] data = null; - if (_Kvpbase.ReadObject(_KvpbaseSettings.Container, id, out data)) return data; + byte[] data = null; + if (_Kvpbase.ReadObject(_KvpbaseSettings.Container, key, out data)) return data; + else throw new IOException("Unable to read object."); + } + + private bool KvpbaseGet(string key, out long contentLength, out Stream stream) + { + contentLength = 0; + stream = null; + if (_Kvpbase.ReadObject(_KvpbaseSettings.Container, key, out contentLength, out stream)) return true; else throw new IOException("Unable to read object."); } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task KvpbaseExists(string id) + private async Task KvpbaseExists(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - return _Kvpbase.ObjectExists(_KvpbaseSettings.Container, id); + return _Kvpbase.ObjectExists(_KvpbaseSettings.Container, key); } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task KvpbaseWrite(string id, string contentType, byte[] data) + private async Task KvpbaseWrite(string key, string contentType, byte[] data) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - return _Kvpbase.WriteObject(_KvpbaseSettings.Container, id, contentType, data); + return _Kvpbase.WriteObject(_KvpbaseSettings.Container, key, contentType, data); + } + + private bool KvpbaseWrite(string key, string contentType, long contentLength, Stream stream) + { + return _Kvpbase.WriteObject(_KvpbaseSettings.Container, key, contentType, contentLength, stream); + } + + private bool KvpbaseGetMetadata(string key, out BlobMetadata md) + { + md = new BlobMetadata(); + md.Key = key; + + ObjectMetadata objMd = null; + if (_Kvpbase.GetObjectMetadata(_KvpbaseSettings.Container, key, out objMd)) + { + md.ContentLength = Convert.ToInt64(objMd.ContentLength); + md.ContentType = objMd.ContentType; + md.ETag = objMd.Md5; + md.Created = objMd.CreatedUtc.Value; + return true; + } + else + { + return false; + } + } + + private bool KvpbaseEnumerate(string continuationToken, out string nextContinuationToken, out List blobs) + { + blobs = new List(); + nextContinuationToken = null; + + int startIndex = 0; + int count = 1000; + if (!String.IsNullOrEmpty(continuationToken)) + { + if (!KvpbaseParseContinuationToken(continuationToken, out startIndex, out count)) + { + return false; + } + } + + ContainerMetadata cmd = null; + if (!_Kvpbase.EnumerateContainer(_KvpbaseSettings.Container, startIndex, count, out cmd)) + { + return false; + } + + if (cmd.Objects != null && cmd.Objects.Count > 0) + { + foreach (ObjectMetadata curr in cmd.Objects) + { + BlobMetadata md = new BlobMetadata(); + md.Key = curr.Key; + md.ETag = curr.Md5; + md.ContentLength = Convert.ToInt64(curr.ContentLength); + md.ContentType = curr.ContentType; + md.Created = curr.CreatedUtc.Value; + blobs.Add(md); + } + } + + return true; + } + + private bool KvpbaseParseContinuationToken(string continuationToken, out int start, out int count) + { + start = -1; + count = -1; + if (String.IsNullOrEmpty(continuationToken)) return false; + byte[] encoded = Convert.FromBase64String(continuationToken); + string encodedStr = Encoding.UTF8.GetString(encoded); + string[] parts = encodedStr.Split(' '); + if (parts.Length != 2) return false; + + if (!Int32.TryParse(parts[0], out start)) return false; + if (!Int32.TryParse(parts[1], out count)) return false; + return true; + } + + private string KvpbaseBuildContinuationToken(long start, int count) + { + string ret = start.ToString() + " " + count.ToString(); + byte[] retBytes = Encoding.UTF8.GetBytes(ret); + return Convert.ToBase64String(retBytes); } #endregion @@ -294,80 +508,231 @@ private async Task KvpbaseWrite(string id, string contentType, byte[] data #region Private-Disk-Methods #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task DiskDelete(string id) + private async Task DiskDelete(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - File.Delete(DiskGenerateUrl(id)); - return true; + try + { + File.Delete(DiskGenerateUrl(key)); + return true; + } + catch (Exception) + { + return false; + } } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task DiskGet(string id) + private async Task DiskGet(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { try { - return File.ReadAllBytes(DiskGenerateUrl(id)); + return File.ReadAllBytes(DiskGenerateUrl(key)); } catch (Exception) { throw new IOException("Unable to read object."); } } + + private bool DiskGet(string key, out long contentLength, out Stream stream) + { + contentLength = 0; + stream = null; + + try + { + string url = DiskGenerateUrl(key); + contentLength = new FileInfo(url).Length; + stream = new FileStream(url, FileMode.Open); + return true; + } + catch (Exception) + { + return false; + } + } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task DiskExists(string id) + private async Task DiskExists(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { - return File.Exists(DiskGenerateUrl(id)); + try + { + return File.Exists(DiskGenerateUrl(key)); + } + catch (Exception) + { + return false; + } } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task DiskWrite(string id, byte[] data) + private async Task DiskWrite(string key, byte[] data) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - { - File.WriteAllBytes(DiskGenerateUrl(id), data); + { + try + { + File.WriteAllBytes(DiskGenerateUrl(key), data); + return true; + } + catch (Exception) + { + return false; + } + } + + private bool DiskWrite(string key, long contentLength, Stream stream) + { + try + { + int bytesRead = 0; + long bytesRemaining = contentLength; + byte[] buffer = new byte[65536]; + string url = DiskGenerateUrl(key); + + using (FileStream fs = new FileStream(url, FileMode.OpenOrCreate)) + { + while (bytesRemaining > 0) + { + bytesRead = stream.Read(buffer, 0, buffer.Length); + if (bytesRead > 0) + { + fs.Write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + } + + return true; + } + catch (Exception) + { + return false; + } + } + + private bool DiskGetMetadata(string key, out BlobMetadata md) + { + string url = DiskGenerateUrl(key); + + FileInfo fi = new FileInfo(url); + md = new BlobMetadata(); + md.Key = key; + md.ContentLength = fi.Length; + md.Created = fi.CreationTimeUtc; + + return true; + } + + private bool DiskEnumerate(string continuationToken, out string nextContinuationToken, out List blobs) + { + nextContinuationToken = null; + blobs = new List(); + + int startIndex = 0; + int count = 1000; + + if (!String.IsNullOrEmpty(continuationToken)) + { + if (!DiskParseContinuationToken(continuationToken, out startIndex, out count)) + { + return false; + } + } + + long maxIndex = startIndex + count; + + long currCount = 0; + IEnumerable files = Directory.EnumerateFiles(_DiskSettings.Directory, "*", SearchOption.TopDirectoryOnly); + files = files.Skip(startIndex).Take(count); + + if (files.Count() < 1) return true; + + nextContinuationToken = DiskBuildContinuationToken(startIndex + count, count); + + foreach (string file in files) + { + string key = Path.GetFileName(file); + FileInfo fi = new FileInfo(file); + + BlobMetadata md = new BlobMetadata(); + md.Key = key; + md.ContentLength = fi.Length; + md.Created = fi.CreationTimeUtc; + blobs.Add(md); + + currCount++; + continue; + } + return true; } - private string DiskGenerateUrl(string id) + private bool DiskParseContinuationToken(string continuationToken, out int start, out int count) + { + start = -1; + count = -1; + if (String.IsNullOrEmpty(continuationToken)) return false; + byte[] encoded = Convert.FromBase64String(continuationToken); + string encodedStr = Encoding.UTF8.GetString(encoded); + string[] parts = encodedStr.Split(' '); + if (parts.Length != 2) return false; + + if (!Int32.TryParse(parts[0], out start)) return false; + if (!Int32.TryParse(parts[1], out count)) return false; + return true; + } + + private string DiskBuildContinuationToken(int start, int count) + { + string ret = start.ToString() + " " + count.ToString(); + byte[] retBytes = Encoding.UTF8.GetBytes(ret); + return Convert.ToBase64String(retBytes); + } + + private string DiskGenerateUrl(string key) { - return _DiskSettings.Directory + "/" + id; + return _DiskSettings.Directory + "/" + key; } #endregion #region Private-S3-Methods - private async Task S3Delete(string id) - { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); - - DeleteObjectRequest request = new DeleteObjectRequest + private async Task S3Delete(string key) + { + try { - BucketName = _AwsSettings.Bucket, - Key = id - }; - - DeleteObjectResponse response = await _S3Client.DeleteObjectAsync(request); - int statusCode = (int)response.HttpStatusCode; + DeleteObjectRequest request = new DeleteObjectRequest + { + BucketName = _AwsSettings.Bucket, + Key = key + }; + + DeleteObjectResponse response = await _S3Client.DeleteObjectAsync(request); + int statusCode = (int)response.HttpStatusCode; - if (response != null) return true; - else return false; + if (response != null) return true; + else return false; + } + catch (Exception) + { + return false; + } } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task S3Get(string id) + private async Task S3Get(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); - + { try { GetObjectRequest request = new GetObjectRequest { BucketName = _AwsSettings.Bucket, - Key = id, + Key = key, }; using (GetObjectResponse response = await _S3Client.GetObjectAsync(request)) @@ -392,21 +757,53 @@ private async Task S3Get(string id) } } catch (Exception) - { + { throw new IOException("Unable to read object."); } } - - private async Task S3Exists(string id) + + private bool S3Get(string key, out long contentLength, out Stream stream) { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); + contentLength = 0; + stream = null; + try + { + GetObjectRequest request = new GetObjectRequest + { + BucketName = _AwsSettings.Bucket, + Key = key, + }; + + GetObjectResponse response = _S3Client.GetObjectAsync(request).Result; + + if (response.ContentLength > 0) + { + contentLength = response.ContentLength; + stream = response.ResponseStream; + return true; + } + else + { + contentLength = 0; + stream = null; + return false; + } + } + catch (Exception) + { + return false; + } + } + + private async Task S3Exists(string key) + { try { GetObjectMetadataRequest request = new GetObjectMetadataRequest { BucketName = _AwsSettings.Bucket, - Key = id + Key = key }; GetObjectMetadataResponse response = await _S3Client.GetObjectMetadataAsync(request); @@ -418,33 +815,136 @@ private async Task S3Exists(string id) } } - private async Task S3Write(string id, string contentType, byte[] data) + private async Task S3Write(string key, string contentType, byte[] data) { - if (String.IsNullOrEmpty(id)) throw new ArgumentNullException(nameof(id)); - - Stream s = new MemoryStream(data); + try + { + Stream s = new MemoryStream(data); + + PutObjectRequest request = new PutObjectRequest + { + BucketName = _AwsSettings.Bucket, + Key = key, + InputStream = s, + ContentType = contentType, + UseChunkEncoding = false + }; + + PutObjectResponse response = await _S3Client.PutObjectAsync(request); + int statusCode = (int)response.HttpStatusCode; + + if (response != null) return true; + else return false; + } + catch (Exception) + { + return false; + } + } - PutObjectRequest request = new PutObjectRequest + private bool S3Write(string key, string contentType, long contentLength, Stream stream) + { + try { - BucketName = _AwsSettings.Bucket, - Key = id, - InputStream = s, - ContentType = contentType, - UseChunkEncoding = false - }; + PutObjectRequest request = new PutObjectRequest + { + BucketName = _AwsSettings.Bucket, + Key = key, + InputStream = stream, + ContentType = contentType, + UseChunkEncoding = false + }; - PutObjectResponse response = await _S3Client.PutObjectAsync(request); - int statusCode = (int)response.HttpStatusCode; + PutObjectResponse response = _S3Client.PutObjectAsync(request).Result; + int statusCode = (int)response.HttpStatusCode; - if (response != null) return true; - else return false; + if (response != null) return true; + else return false; + } + catch (Exception) + { + return false; + } } - private string S3GenerateUrl(string id) + private bool S3GetMetadata(string key, out BlobMetadata md) + { + md = new BlobMetadata(); + md.Key = key; + + try + { + GetObjectMetadataRequest request = new GetObjectMetadataRequest(); + request.BucketName = _AwsSettings.Bucket; + request.Key = key; + + GetObjectMetadataResponse response = _S3Client.GetObjectMetadataAsync(request).Result; + + if (response.ContentLength > 0) + { + md.ContentLength = response.ContentLength; + md.ContentType = response.Headers.ContentType; + md.ETag = response.ETag; + md.Created = response.LastModified; + + if (!String.IsNullOrEmpty(md.ETag)) + { + while (md.ETag.Contains("\"")) md.ETag = md.ETag.Replace("\"", ""); + } + + return true; + } + else + { + return false; + } + } + catch (Exception) + { + return false; + } + } + + private bool S3Enumerate(string continuationToken, out string nextContinuationToken, out List blobs) + { + nextContinuationToken = null; + blobs = new List(); + + ListObjectsRequest req = new ListObjectsRequest(); + req.BucketName = _AwsSettings.Bucket; + + if (!String.IsNullOrEmpty(continuationToken)) req.Marker = continuationToken; + + ListObjectsResponse resp = _S3Client.ListObjectsAsync(req).Result; + if (resp.S3Objects != null && resp.S3Objects.Count > 0) + { + foreach (S3Object curr in resp.S3Objects) + { + BlobMetadata md = new BlobMetadata(); + md.Key = curr.Key; + md.ContentLength = curr.Size; + md.ETag = curr.ETag; + md.Created = curr.LastModified; + + if (!String.IsNullOrEmpty(md.ETag)) + { + while (md.ETag.Contains("\"")) md.ETag = md.ETag.Replace("\"", ""); + } + + blobs.Add(md); + } + } + + if (!String.IsNullOrEmpty(resp.NextMarker)) nextContinuationToken = resp.NextMarker; + + return true; + } + + private string S3GenerateUrl(string key) { GetPreSignedUrlRequest request = new GetPreSignedUrlRequest(); request.BucketName = _AwsSettings.Bucket; - request.Key = id; + request.Key = key; request.Protocol = Protocol.HTTPS; request.Expires = DateTime.Now.AddYears(100); return _S3Client.GetPreSignedURL(request); @@ -454,11 +954,11 @@ private string S3GenerateUrl(string id) #region Private-Azure-Methods - private async Task AzureDelete(string id) + private async Task AzureDelete(string key) { try { - CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(id); + CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(key); OperationContext ctx = new OperationContext(); await blockBlob.DeleteAsync(DeleteSnapshotsOption.None, null, null, ctx); int statusCode = ctx.LastResult.HttpStatusCode; @@ -470,34 +970,56 @@ private async Task AzureDelete(string id) } } - private async Task AzureGet(string id) + private async Task AzureGet(string key) { byte[] data = null; try { - CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(id); + CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(key); OperationContext ctx = new OperationContext(); MemoryStream stream = new MemoryStream(); await blockBlob.DownloadToStreamAsync(stream); stream.Seek(0, SeekOrigin.Begin); - data = Common.StreamToBytes(stream); + data = Common.StreamToBytes(stream); return data; } catch (Exception) - { + { throw new IOException("Unable to read object."); } } + private bool AzureGet(string key, out long contentLength, out Stream stream) + { + contentLength = 0; + stream = null; + + try + { + CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(key); + blockBlob.FetchAttributesAsync().Wait(); + contentLength = blockBlob.Properties.Length; + stream = new MemoryStream(); + blockBlob.DownloadToStreamAsync(stream).Wait(); + + stream.Seek(0, SeekOrigin.Begin); + return true; + } + catch (Exception) + { + return false; + } + } + #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task AzureExists(string id) + private async Task AzureExists(string key) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { try { - return _AzureBlobClient.GetContainerReference(_AzureSettings.Container).GetBlockBlobReference(id).ExistsAsync().Result; + return _AzureBlobClient.GetContainerReference(_AzureSettings.Container).GetBlockBlobReference(key).ExistsAsync().Result; } catch (Exception) { @@ -506,15 +1028,31 @@ private async Task AzureExists(string id) } #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - private async Task AzureWrite(string id, string contentType, byte[] data) + private async Task AzureWrite(string key, string contentType, byte[] data) #pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously { try { - CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(id); + CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(key); + blockBlob.Properties.ContentType = contentType; + OperationContext ctx = new OperationContext(); + blockBlob.UploadFromByteArrayAsync(data, 0, data.Length).Wait(); + return true; + } + catch (Exception) + { + return false; + } + } + + private bool AzureWrite(string key, string contentType, long contentLength, Stream stream) + { + try + { + CloudBlockBlob blockBlob = _AzureContainer.GetBlockBlobReference(key); blockBlob.Properties.ContentType = contentType; OperationContext ctx = new OperationContext(); - blockBlob.UploadFromByteArrayAsync(data, 0, data.Length).Wait(); + blockBlob.UploadFromStreamAsync(stream, contentLength).Wait(); return true; } catch (Exception) @@ -523,14 +1061,108 @@ private async Task AzureWrite(string id, string contentType, byte[] data) } } - private string AzureGenerateUrl(string id) + private bool AzureGetMetadata(string key, out BlobMetadata md) + { + md = new BlobMetadata(); + md.Key = key; + + try + { + CloudBlobContainer container = _AzureBlobClient.GetContainerReference(_AzureSettings.Container); + CloudBlockBlob blockBlob = container.GetBlockBlobReference(key); + blockBlob.FetchAttributesAsync().Wait(); + md.ContentLength = blockBlob.Properties.Length; + md.ContentType = blockBlob.Properties.ContentType; + md.ETag = blockBlob.Properties.ETag; + md.Created = blockBlob.Properties.Created.Value.UtcDateTime; + + if (!String.IsNullOrEmpty(md.ETag)) + { + while (md.ETag.Contains("\"")) md.ETag = md.ETag.Replace("\"", ""); + } + + return true; + } + catch (Exception) + { + return false; + } + + } + + private bool AzureEnumerate(string continuationToken, out string nextContinuationToken, out List blobs) + { + nextContinuationToken = null; + blobs = new List(); + + BlobContinuationToken bct = null; + if (!String.IsNullOrEmpty(continuationToken)) + { + if (!AzureGetContinuationToken(continuationToken, out bct)) + { + return false; + } + } + + BlobResultSegment segment = _AzureContainer.ListBlobsSegmentedAsync(bct).Result; + if (segment == null || segment.Results == null || segment.Results.Count() < 1) return true; + + foreach (IListBlobItem item in segment.Results) + { + if (item.GetType() == typeof(CloudBlockBlob)) + { + CloudBlockBlob blob = (CloudBlockBlob)item; + BlobMetadata md = new BlobMetadata(); + md.Key = blob.Name; + md.ETag = blob.Properties.ETag; + md.ContentType = blob.Properties.ContentType; + md.ContentLength = blob.Properties.Length; + md.Created = blob.Properties.Created.Value.DateTime; + + if (!String.IsNullOrEmpty(md.ETag)) + { + while (md.ETag.Contains("\"")) md.ETag = md.ETag.Replace("\"", ""); + } + + blobs.Add(md); + } + } + + if (segment.ContinuationToken != null) + { + nextContinuationToken = Guid.NewGuid().ToString(); + AzureStoreContinuationToken(nextContinuationToken, segment.ContinuationToken); + } + + if (!String.IsNullOrEmpty(continuationToken)) AzureRemoveContinuationToken(continuationToken); + + return true; + } + + private void AzureStoreContinuationToken(string guid, BlobContinuationToken token) + { + _AzureContinuationTokens.TryAdd(guid, token); + } + + private bool AzureGetContinuationToken(string guid, out BlobContinuationToken token) + { + return _AzureContinuationTokens.TryGetValue(guid, out token); + } + + private void AzureRemoveContinuationToken(string guid) + { + BlobContinuationToken token = null; + _AzureContinuationTokens.TryRemove(guid, out token); + } + + private string AzureGenerateUrl(string key) { return "https://" + _AzureSettings.AccountName + ".blob.core.windows.net/" + _AzureSettings.Container + "/" + - id; + key; } #endregion diff --git a/README.md b/README.md index 6df8d1e..bac6708 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ If you have any issues or feedback, please file an issue here in Github. We'd lo This project was built to provide a simple interface over external storage to help support projects that need to work with potentially multiple storage providers. It is by no means a comprehensive interface, rather, it supports core methods for creation, retrieval, and deletion. -## New in v1.2.0 +## New in v1.3.x -- Breaking change, add ContentType to Write method -- Added missing AWS regions +- Added enumeration capabilities to list contents of a bucket or container +- Added metadata capabilities to retrieve metadata for a given BLOB ## Example Project @@ -84,6 +84,10 @@ bool success = blobs.Delete("test").Result; New capabilities and fixes starting in v1.0.0 will be shown here. +v1.2.x +- Breaking change, add ContentType to Write method +- Added missing AWS regions + v1.1.x - Breaking change; async methods - Retarget to .NET Core 2.0 and .NET Framework 4.6.2 diff --git a/Test/Program.cs b/Test/Program.cs index 5a7b7a0..3f38c38 100644 --- a/Test/Program.cs +++ b/Test/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -23,10 +24,7 @@ static void Main(string[] args) bool runForever = true; while (runForever) - { - byte[] data = null; - bool success = false; - + { string cmd = InputString("Command [? for help]:", null, false); switch (cmd) { @@ -42,26 +40,28 @@ static void Main(string[] args) Console.Clear(); break; case "get": - data = _Blobs.Get(InputString("ID:", null, false)).Result; - if (data != null && data.Length > 0) - { - Console.WriteLine(Encoding.UTF8.GetString(data)); - } + ReadBlob(); break; case "write": - success = _Blobs.Write( - InputString("ID:", null, false), - InputString("Content type:", "text/plain", false), - Encoding.UTF8.GetBytes(InputString("Data:", null, false))).Result; - Console.WriteLine("Success: " + success); + WriteBlob(); break; case "del": - success = _Blobs.Delete( - InputString("ID:", null, false)).Result; - Console.WriteLine("Success: " + success); + DeleteBlob(); + break; + case "upload": + UploadBlob(); + break; + case "download": + DownloadBlob(); break; case "exists": - Console.WriteLine(_Blobs.Exists(InputString("ID:", null, false)).Result); + BlobExists(); + break; + case "md": + BlobMetadata(); + break; + case "enum": + Enumerate(); break; } } @@ -163,13 +163,145 @@ static string InputString(string question, string defaultAnswer, bool allowNull) static void Menu() { Console.WriteLine("Available commands:"); - Console.WriteLine(" ? help, this menu"); - Console.WriteLine(" cls clear the screen"); - Console.WriteLine(" q quit"); - Console.WriteLine(" get get a BLOB"); - Console.WriteLine(" write write a BLOB"); - Console.WriteLine(" del delete a BLOB"); - Console.WriteLine(" exists check if a BLOB exists"); + Console.WriteLine(" ? Help, this menu"); + Console.WriteLine(" cls Clear the screen"); + Console.WriteLine(" q Quit"); + Console.WriteLine(" get Get a BLOB"); + Console.WriteLine(" write Write a BLOB"); + Console.WriteLine(" del Delete a BLOB"); + Console.WriteLine(" upload Upload a BLOB from a file"); + Console.WriteLine(" download Download a BLOB from a file"); + Console.WriteLine(" exists Check if a BLOB exists"); + Console.WriteLine(" md Retrieve BLOB metadata"); + Console.WriteLine(" enum Enumerate a bucket"); + } + + static void WriteBlob() + { + bool success = _Blobs.Write( + InputString("Key:", null, false), + InputString("Content type:", "text/plain", false), + Encoding.UTF8.GetBytes(InputString("Data:", null, false))).Result; + Console.WriteLine("Success: " + success); + } + + static void ReadBlob() + { + byte[] data = _Blobs.Get(InputString("Key:", null, false)).Result; + if (data != null && data.Length > 0) + { + Console.WriteLine(Encoding.UTF8.GetString(data)); + } + } + + static void DeleteBlob() + { + bool success = _Blobs.Delete( + InputString("Key:", null, false)).Result; + Console.WriteLine("Success: " + success); + } + + static void BlobExists() + { + Console.WriteLine(_Blobs.Exists(InputString("Key:", null, false)).Result); + } + + static void UploadBlob() + { + string filename = InputString("Filename:", null, false); + string key = InputString("Key:", null, false); + string contentType = InputString("Content type:", null, true); + + FileInfo fi = new FileInfo(filename); + long contentLength = fi.Length; + + using (FileStream fs = new FileStream(filename, FileMode.Open)) + { + if (_Blobs.Write(key, contentType, contentLength, fs)) + { + Console.WriteLine("Success"); + } + else + { + Console.WriteLine("Failed"); + } + } + } + + static void DownloadBlob() + { + string key = InputString("Key:", null, false); + string filename = InputString("Filename:", null, false); + + long contentLength = 0; + Stream stream = null; + + if (_Blobs.Get(key, out contentLength, out stream)) + { + using (FileStream fs = new FileStream(filename, FileMode.OpenOrCreate)) + { + int bytesRead = 0; + long bytesRemaining = contentLength; + byte[] buffer = new byte[65536]; + + while (bytesRemaining > 0) + { + bytesRead = stream.Read(buffer, 0, buffer.Length); + if (bytesRead > 0) + { + fs.Write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + } + + Console.WriteLine("Success"); + } + else + { + Console.WriteLine("Failed"); + } + } + + static void BlobMetadata() + { + BlobMetadata md = null; + bool success = _Blobs.GetMetadata( + InputString("Key:", null, false), + out md); + if (success) + { + Console.WriteLine(md.ToString()); + } + } + + static void Enumerate() + { + List blobs = null; + string nextContinuationToken = null; + + if (_Blobs.Enumerate( + InputString("Continuation token:", null, true), + out nextContinuationToken, + out blobs)) + { + if (blobs != null && blobs.Count > 0) + { + foreach (BlobMetadata curr in blobs) + { + Console.WriteLine( + String.Format("{0,-27}", curr.Key) + + String.Format("{0,-18}", curr.ContentLength.ToString() + " bytes") + + String.Format("{0,-30}", curr.Created.ToString("yyyy-MM-dd HH:mm:ss"))); + } + } + else + { + Console.WriteLine("(none)"); + } + + if (!String.IsNullOrEmpty(nextContinuationToken)) Console.WriteLine("Continuation token: " + nextContinuationToken); + } } } }