diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleConverter.asmdef b/asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleConverter.asmdef index d6c8405d..5090dc24 100644 --- a/asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleConverter.asmdef +++ b/asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleConverter.asmdef @@ -26,7 +26,9 @@ "overrideReferences": true, "precompiledReferences": [ "Sentry.dll", - "Unity.Plastic.Newtonsoft.Json.dll" + "Unity.Plastic.Newtonsoft.Json.dll", + "AWSSDK.Core.dll", + "AWSSDK.S3.dll" ], "autoReferenced": true, "defineConstraints": [], diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/Config.cs b/asset-bundle-converter/Assets/AssetBundleConverter/Config.cs index ff3c5a4d..0392c96a 100644 --- a/asset-bundle-converter/Assets/AssetBundleConverter/Config.cs +++ b/asset-bundle-converter/Assets/AssetBundleConverter/Config.cs @@ -35,6 +35,12 @@ public static class Config internal static string[] gltfExtensions = { ".glb", ".gltf" }; internal static string[] textureExtensions = { ".jpg", ".png", ".jpeg", ".tga", ".gif", ".bmp", ".psd", ".tiff", ".iff", ".ktx" }; + internal const string LODS_URL = "lods"; + + + internal const string CLI_BUCKET_DIRECTORY = "bucketDirectory"; + internal const string CLI_BUCKET = "bucket"; + internal static string GetDownloadPath() => PathUtils.FixDirectorySeparator(DOWNLOADED_PATH_ROOT + DASH); } } diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter.meta new file mode 100644 index 00000000..26e067e8 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: b3e63c3bdd1e69a46bdb63d400e75be2 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor.meta new file mode 100644 index 00000000..555c851d --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 94750dc0a97e6234eb814a1707a44741 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor/LODClient.cs b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor/LODClient.cs new file mode 100644 index 00000000..c0ed11a1 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor/LODClient.cs @@ -0,0 +1,241 @@ +using System; +using UnityEngine; +using UnityEditor; +using System.IO; +using System.Collections.Generic; +using AssetBundleConverter.LODsConverter.Utils; +using UnityEngine.Rendering; +using Object = UnityEngine.Object; + +namespace DCL.ABConverter +{ + public class LODClient : MonoBehaviour + { + private static readonly string outputPath = Config.ASSET_BUNDLES_PATH_ROOT + Path.DirectorySeparatorChar; + private static readonly string tempPath = Path.Combine(Application.dataPath, "temp"); + + + [MenuItem("Assets/Export URL LODs")] + public static async void ExportURLLODsToAssetBundles() + { + string[] commandLineArgs = Environment.GetCommandLineArgs(); + + string customOutputDirectory = ""; + string lodsURL = ""; + + if (Utils.ParseOption(commandLineArgs, Config.LODS_URL, 1, out string[] lodsURLArg)) + lodsURL = lodsURLArg[0]; + + if (Utils.ParseOption(commandLineArgs, Config.CLI_SET_CUSTOM_OUTPUT_ROOT_PATH, 1, out string[] outputDirectoryArg)) + customOutputDirectory = outputDirectoryArg[0] + "/"; + else + customOutputDirectory = outputPath; + + Debug.Log("Starting file download"); + var urlFileDownloader = new URLFileDownloader(lodsURL, tempPath); + string[] downloadedFiles = await urlFileDownloader.Download(); + Debug.Log("Finished file download"); + try + { + AssetDatabase.SaveAssets(); + ExportFilesToAssetBundles(downloadedFiles, customOutputDirectory); + Utils.Exit(); + } + catch (Exception e) + { + Utils.Exit(1); + } + } + + + + public static async void ExportS3LODsToAssetBundles() + { + string[] commandLineArgs = Environment.GetCommandLineArgs(); + + string bucketName = ""; + string bucketDirectory = ""; + string customOutputDirectory = ""; + + if (Utils.ParseOption(commandLineArgs, Config.CLI_BUCKET, 1, out string[] bucketArg)) + bucketName = bucketArg[0].ToLower(); + + if (Utils.ParseOption(commandLineArgs, Config.CLI_BUCKET_DIRECTORY, 1, out string[] bucketDirectoryArg)) + bucketDirectory = bucketDirectoryArg[0].ToLower(); + + if (Utils.ParseOption(commandLineArgs, Config.CLI_SET_CUSTOM_OUTPUT_ROOT_PATH, 1, out string[] outputDirectoryArg)) + customOutputDirectory = outputDirectoryArg[0].ToLower(); + else + customOutputDirectory = outputPath; + + var amazonS3FileProvider = new AmazonS3FileProvider("lods", "LOD/Sources/1707159813915/", tempPath); + await amazonS3FileProvider.Download(); + + //string[] downloadedFiles = Array.Empty(); + //ExportFilesToAssetBundles(downloadedFiles, customOutputDirectory); + } + + [MenuItem("Assets/Export Asset Bundles")] + private static void ExportAssetBundles() + { + BuildPipeline.BuildAssetBundles(outputPath, BuildAssetBundleOptions.None, EditorUserBuildSettings.activeBuildTarget); + } + + [MenuItem("Assets/Export FBX To Asset Bundles")] + private static void ExportFBXToAssetBundles() + { + string[] fileEntries = Directory.GetFiles(Path.Combine(Application.dataPath, "ExportToAssetBundle"), "*.fbx", SearchOption.AllDirectories); + ExportFilesToAssetBundles(fileEntries, outputPath); + } + + private static void ExportFilesToAssetBundles(string[] filesToExport, string outputPath) + { + Directory.CreateDirectory(outputPath); + Directory.CreateDirectory(tempPath); + + foreach (string fileName in filesToExport) + { + string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName); + string assetBundlePath = Path.Combine(outputPath, fileNameWithoutExtension.ToLower()); + if (File.Exists(assetBundlePath)) + continue; + + string newPath = LODUtils.MoveFileToMatchingFolder(fileName); + + //Get the relative path from the Assets folder + ProcessModel(PathUtils.GetRelativePathTo(Application.dataPath, newPath), tempPath); + GC.Collect(); + } + + + BuildPipeline.BuildAssetBundles(outputPath, BuildAssetBundleOptions.None, EditorUserBuildSettings.activeBuildTarget); + Directory.Delete(tempPath, true); + Debug.Log("Conversion done"); + } + + private static void ProcessModel(string fileToProcess, string tempPath) + { + var asset = AssetDatabase.LoadAssetAtPath(fileToProcess); + if (asset == null) return; + + string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileToProcess); + // Extracting textures and materials + var importer = AssetImporter.GetAtPath(fileToProcess) as ModelImporter; + string subTempPath = Path.Combine(tempPath, fileNameWithoutExtension); + Directory.CreateDirectory(subTempPath); + string subTempPathRelativeToAssets = PathUtils.GetRelativePathTo(Application.dataPath, subTempPath); + + if (importer != null) + { + importer.ExtractTextures(Path.GetDirectoryName(fileToProcess)); + importer.materialImportMode = ModelImporterMaterialImportMode.ImportStandard; + AssetDatabase.WriteImportSettingsIfDirty(fileToProcess); + AssetDatabase.ImportAsset(fileToProcess, ImportAssetOptions.ForceUpdate); + + var instantiated = Instantiate(AssetDatabase.LoadAssetAtPath(fileToProcess)); + instantiated.name = fileNameWithoutExtension.ToLower(); + + if (fileToProcess.Contains("_0")) + { + SetDCLShaderMaterial(fileToProcess, instantiated, subTempPathRelativeToAssets, false); + GenerateColliders(fileToProcess, instantiated); + } + else + SetDCLShaderMaterial(fileToProcess, instantiated, subTempPathRelativeToAssets, true); + + + string prefabPath = tempPath + "/" + instantiated + ".prefab"; + PrefabUtility.SaveAsPrefabAsset(instantiated, prefabPath); + DestroyImmediate(instantiated); + + var prefabImporter = AssetImporter.GetAtPath(PathUtils.GetRelativePathTo(Application.dataPath, prefabPath)); + prefabImporter.SetAssetBundleNameAndVariant(fileNameWithoutExtension, ""); + AssetDatabase.Refresh(); + } + } + + private static void GenerateColliders(string path, GameObject instantiated) + { + var meshFilters = instantiated.GetComponentsInChildren(); + + foreach (var filter in meshFilters) + { + if (filter.name.Contains("_collider", StringComparison.OrdinalIgnoreCase)) + ConfigureColliders(filter.transform, filter); + } + + var renderers = instantiated.GetComponentsInChildren(); + + foreach (var r in renderers) + { + if (r.name.Contains("_collider", StringComparison.OrdinalIgnoreCase)) + DestroyImmediate(r); + } + } + + private static void ConfigureColliders(Transform transform, MeshFilter filter) + { + if (filter != null) + { + Physics.BakeMesh(filter.sharedMesh.GetInstanceID(), false); + filter.gameObject.AddComponent(); + DestroyImmediate(filter.GetComponent()); + } + + foreach (Transform child in transform) + { + var f = child.gameObject.GetComponent(); + ConfigureColliders(child, f); + } + } + + private static void SetDCLShaderMaterial(string path, GameObject transform, string tempPath, bool setDefaultTransparency) + { + var childrenRenderers = transform.GetComponentsInChildren(); + var materialsDictionary = new Dictionary(); + foreach (var componentsInChild in childrenRenderers) + { + var savedMaterials = new List(); + for (int i = 0; i < componentsInChild.sharedMaterials.Length; i++) + { + var material = componentsInChild.sharedMaterials[i]; + var duplicatedMaterial = new Material(material); + duplicatedMaterial.shader = Shader.Find("DCL/Scene"); + if (duplicatedMaterial.name.Contains("FORCED_TRANSPARENT")) + ApplyTransparency(duplicatedMaterial, setDefaultTransparency); + + string materialName = $"{duplicatedMaterial.name.Replace("(Instance)", Path.GetFileNameWithoutExtension(path))}.mat"; + if (!materialsDictionary.ContainsKey(materialName)) + { + string materialPath = Path.Combine(tempPath, materialName); + AssetDatabase.CreateAsset(duplicatedMaterial, materialPath); + AssetDatabase.Refresh(); + materialsDictionary.Add(materialName, AssetDatabase.LoadAssetAtPath(materialPath)); + } + + savedMaterials.Add(materialsDictionary[materialName]); + } + + componentsInChild.sharedMaterials = savedMaterials.ToArray(); + } + } + + private static void ApplyTransparency(Material duplicatedMaterial, bool setDefaultTransparency) + { + duplicatedMaterial.EnableKeyword("_ALPHAPREMULTIPLY_ON"); + duplicatedMaterial.EnableKeyword("_SURFACE_TYPE_TRANSPARENT"); + + duplicatedMaterial.SetFloat("_Surface", 1); + duplicatedMaterial.SetFloat("_BlendMode", 0); + duplicatedMaterial.SetFloat("_AlphaCutoffEnable", 0); + duplicatedMaterial.SetFloat("_SrcBlend", 1f); + duplicatedMaterial.SetFloat("_DstBlend", 10f); + duplicatedMaterial.SetFloat("_AlphaSrcBlend", 1f); + duplicatedMaterial.SetFloat("_AlphaDstBlend", 10f); + duplicatedMaterial.SetFloat("_ZTestDepthEqualForOpaque", 4f); + duplicatedMaterial.renderQueue = (int)RenderQueue.Transparent; + + duplicatedMaterial.color = new Color(duplicatedMaterial.color.r, duplicatedMaterial.color.g, duplicatedMaterial.color.b, setDefaultTransparency ? 0.8f : duplicatedMaterial.color.a); + } + } +} \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor/LODClient.cs.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor/LODClient.cs.meta new file mode 100644 index 00000000..1e4bdd4d --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Editor/LODClient.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 176a2f48b0f825b489908949177aa546 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins.meta new file mode 100644 index 00000000..e36f4615 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: e25291880dfeec94eb90fe24f82bec9f +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.Core.dll b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.Core.dll new file mode 100644 index 00000000..37d412b9 Binary files /dev/null and b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.Core.dll differ diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.Core.dll.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.Core.dll.meta new file mode 100644 index 00000000..91bbaaaa --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.Core.dll.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: eedc8e07c9719a249be250278941ee41 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.S3.dll b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.S3.dll new file mode 100644 index 00000000..92977732 Binary files /dev/null and b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.S3.dll differ diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.S3.dll.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.S3.dll.meta new file mode 100644 index 00000000..07aa56ff --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/AWSSDK.S3.dll.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: 66dceb464007f304abf91f69a8b96acd +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/Microsoft.Bcl.AsyncInterfaces.dll b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/Microsoft.Bcl.AsyncInterfaces.dll new file mode 100644 index 00000000..421e8124 Binary files /dev/null and b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/Microsoft.Bcl.AsyncInterfaces.dll differ diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/Microsoft.Bcl.AsyncInterfaces.dll.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/Microsoft.Bcl.AsyncInterfaces.dll.meta new file mode 100644 index 00000000..f2bd75bc --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/Microsoft.Bcl.AsyncInterfaces.dll.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: f23c287927a2cb84b980de571f186b72 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Runtime.CompilerServices.Unsafe.dll b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Runtime.CompilerServices.Unsafe.dll new file mode 100644 index 00000000..491a80a9 Binary files /dev/null and b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Runtime.CompilerServices.Unsafe.dll differ diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Runtime.CompilerServices.Unsafe.dll.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Runtime.CompilerServices.Unsafe.dll.meta new file mode 100644 index 00000000..9f58a780 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Runtime.CompilerServices.Unsafe.dll.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: 111145936f792fa4093143fa6ede8433 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Threading.Tasks.Extensions.dll b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Threading.Tasks.Extensions.dll new file mode 100644 index 00000000..dfab2347 Binary files /dev/null and b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Threading.Tasks.Extensions.dll differ diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Threading.Tasks.Extensions.dll.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Threading.Tasks.Extensions.dll.meta new file mode 100644 index 00000000..9444a047 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Plugins/System.Threading.Tasks.Extensions.dll.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: 6875b02ac884ae04c97c63b08167cf41 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils.meta new file mode 100644 index 00000000..ee67b74f --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 7843c221047b42829c903f0a08765692 +timeCreated: 1709131066 \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/AmazonS3FileProvider.cs b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/AmazonS3FileProvider.cs new file mode 100644 index 00000000..11f6e58d --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/AmazonS3FileProvider.cs @@ -0,0 +1,96 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Amazon; +using Amazon.S3; +using Amazon.S3.Model; +using JetBrains.Annotations; + +namespace AssetBundleConverter.LODsConverter.Utils +{ + public class AmazonS3FileProvider : IFileDownloader + { + private readonly string bucketName = "your-bucket-name"; + private readonly string directoryPath = "your-directory-path/"; // Ensure it ends with a '/' + private readonly string outputPath = "your-output-path"; + private static readonly RegionEndpoint bucketRegion = RegionEndpoint.USEast1; // Update to your bucket's region + private readonly IAmazonS3 s3Client; + + public AmazonS3FileProvider(string bucketName, string bucketDirectory, string outputPath) + { + var config = new AmazonS3Config(); + config.RegionEndpoint = bucketRegion; + config.ServiceURL = "http://localhost:4566"; // Ensure to use the correct endpoint + s3Client = new AmazonS3Client(config); + this.bucketName = bucketName; + directoryPath = bucketDirectory; + this.outputPath = outputPath; + } + + [CanBeNull] + public async Task Download() + { + try + { + var request = new ListObjectsV2Request + { + BucketName = bucketName, Prefix = directoryPath + }; + + ListObjectsV2Response response; + do + { + response = await s3Client.ListObjectsV2Async(request); + foreach (var entry in response.S3Objects) + { + Console.WriteLine($"Downloading {entry.Key}..."); + await DownloadFileAsync(entry.Key); + } + + request.ContinuationToken = response.NextContinuationToken; + } while (response.IsTruncated); + } + catch (AmazonS3Exception e) + { + Console.WriteLine($"Error encountered on server. Message:'{e.Message}' when listing objects"); + } + catch (Exception e) + { + Console.WriteLine($"Unknown encountered on server. Message:'{e.Message}' when listing objects"); + } + + return Array.Empty(); + } + + private async Task DownloadFileAsync(string keyName) + { + string filePath = Path.Combine(outputPath, keyName.Replace("/", "\\")); + // Ensure the directory exists + Directory.CreateDirectory(Path.GetDirectoryName(filePath)); + + try + { + var request = new GetObjectRequest + { + BucketName = bucketName, Key = keyName + }; + + using (var response = await s3Client.GetObjectAsync(request)) + using (var responseStream = response.ResponseStream) + using (var fileStream = File.Create(filePath)) + { + await responseStream.CopyToAsync(fileStream); + Console.WriteLine($"{keyName} has been downloaded to {filePath}"); + } + } + catch (AmazonS3Exception e) + { + Console.WriteLine($"Error encountered on server. Message:'{e.Message}' when downloading an object"); + } + catch (Exception e) + { + Console.WriteLine($"Unknown encountered on server. Message:'{e.Message}' when downloading an object"); + } + } + } +} \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/AmazonS3FileProvider.cs.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/AmazonS3FileProvider.cs.meta new file mode 100644 index 00000000..31929043 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/AmazonS3FileProvider.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a7ff6634051ff9e4f80c69e8ccc3eed6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/IFileDownloader.cs b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/IFileDownloader.cs new file mode 100644 index 00000000..0f54bdff --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/IFileDownloader.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace AssetBundleConverter.LODsConverter.Utils +{ + public interface IFileDownloader + { + Task Download(); + } +} \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/IFileDownloader.cs.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/IFileDownloader.cs.meta new file mode 100644 index 00000000..7eb8e11b --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/IFileDownloader.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: feaeaaba66fd4846896f7db6b60f3c2e +timeCreated: 1709895539 \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/LODUtils.cs b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/LODUtils.cs new file mode 100644 index 00000000..834edcbc --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/LODUtils.cs @@ -0,0 +1,34 @@ +using System; +using System.IO; +using UnityEditor; + +namespace AssetBundleConverter.LODsConverter.Utils +{ + public class LODUtils + { + public static string MoveFileToMatchingFolder(string filePath) + { + string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(filePath); + string currentFolderPath = Path.GetDirectoryName(filePath); + + if (currentFolderPath.EndsWith(fileNameWithoutExtension)) + { + Console.WriteLine("The file is already in the correct folder."); + return filePath; + } + + string targetFolderPath = Path.Combine(currentFolderPath, fileNameWithoutExtension); + Directory.CreateDirectory(targetFolderPath); + + // Create a new path for the file in the new folder + string newFilePath = Path.Combine(targetFolderPath, Path.GetFileName(filePath)); + + // Move the file to the new folder + File.Move(filePath, newFilePath); + // Save assets and refresh the AssetDatabase + AssetDatabase.SaveAssets(); + AssetDatabase.Refresh(); + return newFilePath; + } + } +} \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/LODUtils.cs.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/LODUtils.cs.meta new file mode 100644 index 00000000..20db790a --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/LODUtils.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: d4971e55942f4300826892cb1c9601e8 +timeCreated: 1709131076 \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/URLFileDownloader.cs b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/URLFileDownloader.cs new file mode 100644 index 00000000..7b4a0001 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/URLFileDownloader.cs @@ -0,0 +1,54 @@ +using System.IO; +using System.Threading.Tasks; +using Cysharp.Threading.Tasks; +using UnityEngine; +using UnityEngine.Networking; + +namespace AssetBundleConverter.LODsConverter.Utils +{ + public class URLFileDownloader : IFileDownloader + { + private readonly string lodsURL; + private readonly string tempDownloadPath; + + public URLFileDownloader(string lodsURL, string tempDownloadPath) + { + this.lodsURL = lodsURL; + this.tempDownloadPath = tempDownloadPath; + } + + public async Task Download() + { + Directory.CreateDirectory(tempDownloadPath); + string[] filesToDownload = lodsURL.Split(";"); + string[] downloadedPaths = new string[filesToDownload.Length]; + for (int index = 0; index < filesToDownload.Length; index++) + { + string url = filesToDownload[index]; + using (var webRequest = UnityWebRequest.Get(url)) + { + string fileName = Path.GetFileName(url); + string savePath = Path.Combine(tempDownloadPath, fileName); + Debug.Log($"Starting file download {url}"); + await webRequest.SendWebRequest(); + + if (webRequest.result == UnityWebRequest.Result.Success) + { + // Success, save the downloaded file + File.WriteAllBytes(savePath, webRequest.downloadHandler.data); + Debug.Log($"File downloaded and saved to {savePath}"); + downloadedPaths[index] = savePath; + } + else + { + DCL.ABConverter.Utils.Exit(1); + Debug.LogError($"Error downloading {url}: {webRequest.error}"); + return null; + } + } + } + + return downloadedPaths; + } + } +} \ No newline at end of file diff --git a/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/URLFileDownloader.cs.meta b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/URLFileDownloader.cs.meta new file mode 100644 index 00000000..c211edd6 --- /dev/null +++ b/asset-bundle-converter/Assets/AssetBundleConverter/LODsConverter/Utils/URLFileDownloader.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 39715cd6c0fd4dd796df80e6bb5b1450 +timeCreated: 1709895426 \ No newline at end of file diff --git a/asset-bundle-converter/Packages/manifest.json b/asset-bundle-converter/Packages/manifest.json index 953e56e8..36fa767c 100644 --- a/asset-bundle-converter/Packages/manifest.json +++ b/asset-bundle-converter/Packages/manifest.json @@ -4,6 +4,7 @@ "com.atteneder.ktx": "2.2.3", "com.cysharp.unitask": "https://github.com/Cysharp/UniTask.git?path=src/UniTask/Assets/Plugins/UniTask", "com.unity.ai.navigation": "1.1.5", + "com.unity.assetbundlebrowser": "https://github.com/Unity-Technologies/AssetBundles-Browser.git", "com.unity.collab-proxy": "2.2.0", "com.unity.ide.rider": "3.0.25", "com.unity.ide.visualstudio": "2.0.21", diff --git a/asset-bundle-converter/Packages/packages-lock.json b/asset-bundle-converter/Packages/packages-lock.json index b0cc7426..f6f12b10 100644 --- a/asset-bundle-converter/Packages/packages-lock.json +++ b/asset-bundle-converter/Packages/packages-lock.json @@ -34,6 +34,13 @@ }, "url": "https://packages.unity.com" }, + "com.unity.assetbundlebrowser": { + "version": "https://github.com/Unity-Technologies/AssetBundles-Browser.git", + "depth": 0, + "source": "git", + "dependencies": {}, + "hash": "ad2a81ec3068eafd48753df1825a8dd7201ce60c" + }, "com.unity.burst": { "version": "1.8.9", "depth": 1, diff --git a/consumer-server/src/components.ts b/consumer-server/src/components.ts index d0c5d6ce..9867ef9e 100644 --- a/consumer-server/src/components.ts +++ b/consumer-server/src/components.ts @@ -30,8 +30,8 @@ export async function initComponents(): Promise { const sqsQueue = await config.getString('TASK_QUEUE') const taskQueue = sqsQueue ? - createSqsAdapter({ logs, metrics }, { queueUrl: sqsQueue, queueRegion: AWS_REGION }) : - createMemoryQueueAdapter({ logs, metrics }, { queueName: "ConversionTaskQueue" }) + createSqsAdapter({ logs, metrics }, { queueUrl: sqsQueue, queueRegion: AWS_REGION }) : + createMemoryQueueAdapter({ logs, metrics }, { queueName: "ConversionTaskQueue" }) const s3Bucket = await config.getString('CDN_BUCKET') const cdnS3 = s3Bucket ? new AWS.S3({}) : new MockAws.S3({}) diff --git a/consumer-server/src/controllers/handlers/queue-conversion-handle.ts b/consumer-server/src/controllers/handlers/queue-conversion-handle.ts index 89e87f0f..69a4064f 100644 --- a/consumer-server/src/controllers/handlers/queue-conversion-handle.ts +++ b/consumer-server/src/controllers/handlers/queue-conversion-handle.ts @@ -16,7 +16,7 @@ export async function queueTaskHandler(context: HandlerContextWithPath<"metrics" if (!DeploymentToSqs.validate(body)) return { status: 403, body: { errors: DeploymentToSqs.validate.errors } } - const message = await taskQueue.publish(body) + const message = await taskQueue.publish(body as DeploymentToSqs & { lods: string[] | undefined; }) return { status: 201, diff --git a/consumer-server/src/logic/conversion-task.ts b/consumer-server/src/logic/conversion-task.ts index 8ca4581b..4e42dab2 100644 --- a/consumer-server/src/logic/conversion-task.ts +++ b/consumer-server/src/logic/conversion-task.ts @@ -3,7 +3,7 @@ import { FileVariant } from '@dcl/cdn-uploader/dist/types' import * as promises from 'fs/promises' import { rimraf } from 'rimraf' import { AppComponents } from '../types' -import { runConversion } from './run-conversion' +import { runConversion, runLodsConversion } from './run-conversion' type Manifest = { version: string @@ -73,6 +73,115 @@ function getAbVersionEnvName(buildTarget: string) } } +export async function executeLODConversion(components: Pick, entityId: string, lods: string[]) { + const $LOD_BUCKET = await components.config.getString('LOD_BUCKET') + if (!$LOD_BUCKET) { + throw new Error('LOD_BUCKET is not defined') + } + + const $LOGS_BUCKET = await components.config.getString('LOGS_BUCKET') + const $UNITY_PATH = await components.config.requireString('UNITY_PATH') + const $PROJECT_PATH = await components.config.requireString('PROJECT_PATH') + const $BUILD_TARGET = await components.config.requireString('BUILD_TARGET') + + const abVersionEnvName = getAbVersionEnvName($BUILD_TARGET) + const $AB_VERSION = await components.config.requireString(abVersionEnvName) + const logger = components.logs.getLogger(`ExecuteConversion`) + + const cdnBucket = await getCdnBucket(components) + const logFile = `/tmp/asset_bundles_logs/export_log_${entityId}_${Date.now()}.txt` + const s3LogKey = `logs/${$AB_VERSION}/${entityId}/${new Date().toISOString()}.txt` + const outDirectory = `/tmp/asset_bundles_contents/entity_${entityId}` + let defaultLoggerMetadata = { entityId, lods, version: $AB_VERSION, logFile } as any + + logger.info("Starting conversion for " + $BUILD_TARGET, defaultLoggerMetadata) + + try { + const exitCode = await runLodsConversion(logger, components, { + entityId, + logFile, + outDirectory, + lods, + unityPath: $UNITY_PATH, + projectPath: $PROJECT_PATH, + timeout: 60 * 60 * 1000, + }) + + components.metrics.increment('ab_converter_exit_codes', { exit_code: (exitCode ?? -1)?.toString() }) + + const generatedFiles = await promises.readdir(outDirectory) + + if (generatedFiles.length == 0) { + // this is an error, if succeeded, we should see at least a manifest file + components.metrics.increment('ab_converter_empty_conversion', { ab_version: $AB_VERSION }) + logger.error('Empty conversion', { ...defaultLoggerMetadata } as any) + return + } + + await uploadDir(components.cdnS3, cdnBucket, outDirectory, $AB_VERSION, { + concurrency: 10, + matches: [ + { + // the rest of the elements will be uploaded as application/wasm + // to be compressed and cached by cloudflare + match: "**/*", + contentType: "application/wasm", + immutable: true, + variants: [FileVariant.Brotli, FileVariant.Uncompressed], + skipRepeated: true + } + ] + }) + } catch (error: any) { + logger.debug(await promises.readFile(logFile, 'utf8'), defaultLoggerMetadata) + components.metrics.increment('ab_converter_exit_codes', { exit_code: 'FAIL' }) + logger.error(error) + + setTimeout(() => { + // kill the process in one minute, enough time to allow prometheus to collect the metrics + process.exit(199) + }, 60_000) + + throw error + } finally { + if ($LOGS_BUCKET) { + const log = `https://${$LOGS_BUCKET}.s3.amazonaws.com/${s3LogKey}` + + logger.info(`LogFile=${log}`, defaultLoggerMetadata) + await components.cdnS3 + .upload({ + Bucket: $LOGS_BUCKET, + Key: s3LogKey, + Body: await promises.readFile(logFile), + ACL: 'public-read', + }) + .promise() + } else { + logger.info(`!!!!!!!! Log file not deleted or uploaded ${logFile}`, defaultLoggerMetadata) + } + + // delete output files + try { + await rimraf(logFile, { maxRetries: 3 }) + } catch (err: any) { + logger.error(err, defaultLoggerMetadata) + } + try { + await rimraf(outDirectory, { maxRetries: 3 }) + } catch (err: any) { + logger.error(err, defaultLoggerMetadata) + } + // delete library folder + try { + await rimraf(`$PROJECT_PATH/Library`, { maxRetries: 3 }) + } catch (err: any) { + logger.error(err, defaultLoggerMetadata) + } + } + + logger.debug("LOD Conversion finished", defaultLoggerMetadata) +} + export async function executeConversion(components: Pick, entityId: string, contentServerUrl: string, force: boolean | undefined) { const $LOGS_BUCKET = await components.config.getString('LOGS_BUCKET') const $UNITY_PATH = await components.config.requireString('UNITY_PATH') diff --git a/consumer-server/src/logic/run-conversion.ts b/consumer-server/src/logic/run-conversion.ts index 34feb4ac..fd164494 100644 --- a/consumer-server/src/logic/run-conversion.ts +++ b/consumer-server/src/logic/run-conversion.ts @@ -5,6 +5,68 @@ import { dirname } from 'path' import { AppComponents } from '../types' import { execCommand } from './run-command' +async function makeLogFileAndOutputDirectoryAvailable(options: { + logFile: string, + outDirectory: string +}) { + // touch logfile and create folders + await fs.mkdir(dirname(options.logFile), { recursive: true }) + await fs.mkdir(options.outDirectory, { recursive: true }) + closeSync(openSync(options.logFile, 'w')) +} + +async function executeProgram(options: { logger: ILoggerComponent.ILogger, components: Pick, childArg0: string, childArguments: string[], projectPath: string, timeout: number }) { + const { logger, components, childArg0, childArguments, projectPath, timeout } = options + const { exitPromise, child } = execCommand(logger, childArg0, childArguments, process.env as any, projectPath) + + if (timeout) { + setTimeout(() => { + if (exitPromise.isPending) { + try { + if (!child.killed) { + logger.warn('Process did not finish', { pid: child.pid?.toString() || '?', command: childArg0, args: childArguments.join(' ') } as any) + components.metrics.increment('ab_converter_timeout') + exitPromise.reject(new Error('Process did not finish')) + if (!child.kill('SIGKILL')) { + logger.error('Error trying to kill child process', { pid: child.pid?.toString() || '?', command: childArg0, args: childArguments.join(' ') } as any) + } + } + } catch (err: any) { + logger.error(err) + } + } + }, timeout) + } + + return await exitPromise +} + +export async function runLodsConversion(logger: ILoggerComponent.ILogger, components: Pick, options: { + logFile: string, + outDirectory: string, + entityId: string, + lods: string[], + unityPath: string, + projectPath: string, + timeout: number +}) { + makeLogFileAndOutputDirectoryAvailable(options) + + const childArg0 = `${options.unityPath}/Editor/Unity` + + const childArguments: string[] = [ + '-projectPath', options.projectPath, + '-batchmode', + '-executeMethod', 'DCL.ABConverter.LODClient.ExportURLLODsToAssetBundles', + '-sceneCid', options.entityId, + '-logFile', options.logFile, + '-lods', options.lods.join(','), + '-output', options.outDirectory + ] + + return await executeProgram({ logger, components, childArg0, childArguments, projectPath: options.projectPath, timeout: options.timeout }) +} + export async function runConversion( logger: ILoggerComponent.ILogger, components: Pick, @@ -19,10 +81,7 @@ export async function runConversion( unityBuildTarget: string, } ) { - // touch logfile and create folders - await fs.mkdir(dirname(options.logFile), { recursive: true }) - await fs.mkdir(options.outDirectory, { recursive: true }) - closeSync(openSync(options.logFile, 'w')) + makeLogFileAndOutputDirectoryAvailable(options) // normalize content server URL let contentServerUrl = options.contentServerUrl @@ -46,26 +105,5 @@ export async function runConversion( '-buildTarget', options.unityBuildTarget ] - const { exitPromise, child } = execCommand(logger, childArg0, childArguments, process.env as any, options.projectPath) - - if (options.timeout) { - setTimeout(() => { - if (exitPromise.isPending) { - try { - if (!child.killed) { - logger.warn('Process did not finish', { pid: child.pid?.toString() || '?', command: childArg0, args: childArguments.join(' ') } as any) - components.metrics.increment('ab_converter_timeout') - exitPromise.reject(new Error('Process did not finish')) - if (!child.kill('SIGKILL')) { - logger.error('Error trying to kill child process', { pid: child.pid?.toString() || '?', command: childArg0, args: childArguments.join(' ') } as any) - } - } - } catch (err: any) { - logger.error(err) - } - } - }, options.timeout) - } - - return await exitPromise + return await executeProgram({ logger, components, childArg0, childArguments, projectPath: options.projectPath, timeout: options.timeout }) } \ No newline at end of file diff --git a/consumer-server/src/service.ts b/consumer-server/src/service.ts index cd1e6d9c..9a762b5d 100644 --- a/consumer-server/src/service.ts +++ b/consumer-server/src/service.ts @@ -1,6 +1,6 @@ import { ILoggerComponent, Lifecycle } from "@well-known-components/interfaces" import { setupRouter } from "./controllers/routes" -import { executeConversion } from "./logic/conversion-task" +import { executeConversion, executeLODConversion } from "./logic/conversion-task" import checkDiskSpace from 'check-disk-space' import { AppComponents, GlobalContext, TestComponents } from "./types" @@ -36,7 +36,11 @@ export async function main(program: Lifecycle.EntryPointParameters { try { components.metrics.increment('ab_converter_running_conversion') - await executeConversion(components, job.entity.entityId, job.contentServerUrls![0], job.force) + if (job.lods) { + await executeLODConversion(components, job.entity.entityId, job.lods) + } else { + await executeConversion(components, job.entity.entityId, job.contentServerUrls![0], job.force) + } } finally { components.metrics.decrement('ab_converter_running_conversion') } diff --git a/consumer-server/src/types.ts b/consumer-server/src/types.ts index c2e70e63..517728ba 100644 --- a/consumer-server/src/types.ts +++ b/consumer-server/src/types.ts @@ -22,7 +22,7 @@ export type BaseComponents = { logs: ILoggerComponent server: IHttpServerComponent fetch: IFetchComponent - taskQueue: ITaskQueue + taskQueue: ITaskQueue metrics: IMetricsComponent cdnS3: S3 runner: IRunnerComponent diff --git a/windows-test-run-lods.bat b/windows-test-run-lods.bat new file mode 100644 index 00000000..8fbd11d9 --- /dev/null +++ b/windows-test-run-lods.bat @@ -0,0 +1 @@ +"C:\Program Files\Unity\Hub\Editor\2022.3.12f1\Editor\Unity.exe" -projectPath "asset-bundle-converter" -batchmode -executeMethod DCL.ABConverter.LODClient.ExportURLLODsToAssetBundles -lods "https://lods-bucket-ed4300a.s3.amazonaws.com/-17,-21/LOD/Sources/1707776785658/bafkreidnwpjkv3yoxsz6iiqh3fahuec7lfsqtmkyz3yf6dgps454ngldnu_0.fbx;https://lods-bucket-ed4300a.s3.amazonaws.com/-17,-21/LOD/Sources/1707776785658/bafkreidnwpjkv3yoxsz6iiqh3fahuec7lfsqtmkyz3yf6dgps454ngldnu_1.fbx;https://lods-bucket-ed4300a.s3.amazonaws.com/-17,-21/LOD/Sources/1707776785658/bafkreidnwpjkv3yoxsz6iiqh3fahuec7lfsqtmkyz3yf6dgps454ngldnu_2.fbx" -output "C:/Users/juani/Documents/Decentraland/asset-bundle-converter - Copy/AssetBundles" -logFile ./tmp/log.txt \ No newline at end of file