Feat: Scene Shader

.github/workflows/docker.yml

@@ -8,13 +8,13 @@ jobs:
     strategy:
       matrix:
         include:
-        - unity_docker_image: unityci/editor:2021.3.22f1-webgl-1
+        - unity_docker_image: unityci/editor:2022.3.12f1-webgl-3.0.0
           service_name: asset-bundle-converter
           platform_target: webgl
-        - unity_docker_image: unityci/editor:2021.3.22f1-windows-mono-1
+        - unity_docker_image: unityci/editor:2022.3.12f1-windows-mono-3.0.0
           service_name: asset-bundle-converter-windows
           platform_target: windows
-        - unity_docker_image: unityci/editor:2021.3.22f1-mac-mono-1
+        - unity_docker_image: unityci/editor:2022.3.12f1-mac-mono-3.0.0
           service_name: asset-bundle-converter-mac
           platform_target: mac
 

.github/workflows/manual-build-conversion.yml

@@ -23,13 +23,13 @@ jobs:
     strategy:
       matrix:
         include:
-        - unity_docker_image: unityci/editor:2021.3.22f1-webgl-1
+        - unity_docker_image: unityci/editor:2022.3.12f1-webgl-3.0.0
           service_name: asset-bundle-converter
           platform_target: webgl
-        - unity_docker_image: unityci/editor:2021.3.22f1-windows-mono-1
+        - unity_docker_image: unityci/editor:2022.3.12f1-windows-mono-3.0.0
           service_name: asset-bundle-converter-windows
           platform_target: windows
-        - unity_docker_image: unityci/editor:2021.3.22f1-mac-mono-1
+        - unity_docker_image: unityci/editor:2022.3.12f1-mac-mono-3.0.0
           service_name: asset-bundle-converter-mac
           platform_target: mac
 

Dockerfile

@@ -46,9 +46,9 @@ ENV NODE_PATH $NVM_DIR/versions/node/$NODE_VERSION/lib/node_modules
 ENV PATH $NVM_DIR/versions/node/$NODE_VERSION/bin:$PATH
 
 # Change this value ONLY if we have done breaking changes for every material, doing so is VERY costly
-ENV AB_VERSION v11
-ENV AB_VERSION_WINDOWS v14
-ENV AB_VERSION_MAC v15
+ENV AB_VERSION v12
+ENV AB_VERSION_WINDOWS v15
+ENV AB_VERSION_MAC v16
 
 # NODE_ENV is used to configure some runtime options, like JSON logger
 ENV NODE_ENV production

asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleConverter.asmdef

@@ -15,7 +15,8 @@
         "GUID:9887bf5401cdc9140916d3edbea10b69",
         "GUID:f51ebe6a0ceec4240a699833d6309b23",
         "GUID:5c1ff9bd975acec488150743d53a93ca",
-        "GUID:c5ecc461727906345a35491a0440694f"
+        "GUID:c5ecc461727906345a35491a0440694f",
+        "GUID:15fc0a57446b3144c949da3e2b9737a9"
     ],
     "includePlatforms": [
         "Editor"
@@ -24,7 +25,6 @@
     "allowUnsafeCode": false,
     "overrideReferences": true,
     "precompiledReferences": [
-        "Newtonsoft.Json.dll",
         "Sentry.dll",
         "Unity.Plastic.Newtonsoft.Json.dll"
     ],

asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleConverter.cs

@@ -15,6 +15,7 @@
 using UnityEngine;
 using UnityEngine.Networking;
 using UnityEngine.Profiling;
+using UnityEngine.Rendering.Universal;
 using Debug = UnityEngine.Debug;
 using Environment = AssetBundleConverter.Environment;
 using Object = UnityEngine.Object;
@@ -145,6 +146,7 @@ public async Task ConvertAsync(IReadOnlyList<ContentServerUtils.MappingPair> raw
 
             // First step: initialize directories to download the original assets and to store the results
             InitializeDirectoryPaths(settings.clearDirectoriesOnStart, settings.clearDirectoriesOnStart);
+            AdjustRenderingMode(settings.buildTarget);
             env.assetDatabase.Refresh();
 
             await env.editor.Delay(TimeSpan.FromSeconds(0.1f));
@@ -205,7 +207,6 @@ public async Task ConvertAsync(IReadOnlyList<ContentServerUtils.MappingPair> raw
                 bundlesEndTime = EditorApplication.timeSinceStartup;
             }
 
-
             if (isExitForced)
                 return;
 
@@ -219,6 +220,21 @@ public async Task ConvertAsync(IReadOnlyList<ContentServerUtils.MappingPair> raw
             OnFinish();
         }
 
+        private void AdjustRenderingMode(BuildTarget targetPlatform)
+        {
+            var universalRendererData = Resources.FindObjectsOfTypeAll<UniversalRendererData>().First();
+            var rendererDataPath = AssetDatabase.GetAssetPath(universalRendererData);
+
+            universalRendererData.renderingMode = targetPlatform switch
+                                                  {
+                                                      BuildTarget.StandaloneWindows64 or BuildTarget.StandaloneOSX => RenderingMode.ForwardPlus,
+                                                      BuildTarget.WebGL => RenderingMode.Forward,
+                                                      _ => universalRendererData.renderingMode
+                                                  };
+
+            AssetDatabase.ImportAsset(rendererDataPath);
+        }
+
         private void MarkAndBuildForTarget(BuildTarget target)
         {
 
@@ -437,57 +453,38 @@ private void CreateMaterialAsset(Material originalMaterial, string materialRoot,
             string materialPath =
                 PathUtils.GetRelativePathTo(Application.dataPath, string.Concat(materialRoot, matName, ".mat"));
 
-            var previousMat = env.assetDatabase.LoadAssetAtPath<Material>(materialPath);
-
-            if (previousMat != null)
-                env.assetDatabase.DeleteAsset(materialPath);
-
-            Profiler.BeginSample("ExtractEmbedMaterials.CreateAsset");
-            env.assetDatabase.CreateAsset(Object.Instantiate(originalMaterial), materialPath);
-            //env.assetDatabase.SaveAssets();
-            Profiler.EndSample();
-
-            var newMaterial = env.assetDatabase.LoadAssetAtPath<Material>(materialPath);
+            var newMaterial = Object.Instantiate(originalMaterial);
 
             var shader = newMaterial.shader;
 
             if (settings.stripShaders)
-                env.assetDatabase.MarkAssetBundle(env.assetDatabase, shader,
-                    shader.name + "_IGNORE" + PlatformUtils.GetPlatform());
+                env.assetDatabase.AssignAssetBundle(shader);
 
             var textureProperties = GetTextureProperties(shader);
 
             Profiler.BeginSample("ExtractEmbedMaterials.SetTexture");
             for (var i = 0; i < textureProperties.Count; ++i)
             {
                 int propertyId = textureProperties[i];
-                var tex = originalMaterial.GetTexture(propertyId) as Texture2D;
-
-                // we ensure that the property has a valid material
-                if (!tex) continue;
 
                 // we reassign the texture reference
-                string texName = Utils.NicifyName(tex.name);
-                texName = Path.GetFileNameWithoutExtension(texName).ToLowerInvariant();
-
                 var prevTexture = newMaterial.GetTexture(propertyId);
 
-                if (texNameMap.ContainsKey(texName))
-                    newMaterial.SetTexture(propertyId, texNameMap[texName]);
-                else
-                {
-                    if (prevTexture == null)
-                    {
-                        var message =
-                            $"Failed to set texture \"{texName}\" to material \"{matName}\". This will cause white materials";
-                        log.Error(message);
-                        errorReporter.ReportError(message, settings);
-                        ForceExit((int)ErrorCodes.EMBED_MATERIAL_FAILURE);
-                        return;
-                    }
-                }
+                // after copying a material it should hold the reference to the original texture
+                if (!prevTexture) continue;
+
+                string texName = Utils.NicifyName(prevTexture.name);
+                texName = Path.GetFileNameWithoutExtension(texName).ToLowerInvariant();
+
+                if (texNameMap.TryGetValue(texName, out Texture2D tex))
+                    newMaterial.SetTexture(propertyId, tex);
             }
 
+            Profiler.BeginSample("ExtractEmbedMaterials.CreateAsset");
+            env.assetDatabase.CreateAsset(newMaterial, materialPath);
+            //env.assetDatabase.SaveAssets();
+            Profiler.EndSample();
+
             Profiler.EndSample();
 
             EditorUtility.SetDirty(newMaterial);
@@ -727,7 +724,7 @@ public virtual bool BuildAssetBundles(BuildTarget target, out IAssetBundleManife
 
             // 1. Convert flagged folders to asset bundles only to automatically get dependencies for the metadata
             manifest = env.buildPipeline.BuildAssetBundles(settings.finalAssetBundlePath,
-                BuildAssetBundleOptions.UncompressedAssetBundle | BuildAssetBundleOptions.ForceRebuildAssetBundle,
+                BuildAssetBundleOptions.UncompressedAssetBundle | BuildAssetBundleOptions.ForceRebuildAssetBundle | BuildAssetBundleOptions.AssetBundleStripUnityVersion,
                 target);
 
             if (manifest == null)
@@ -752,7 +749,7 @@ public virtual bool BuildAssetBundles(BuildTarget target, out IAssetBundleManife
 
             // 3. Convert flagged folders to asset bundles again but this time they have the metadata file inside
             manifest = env.buildPipeline.BuildAssetBundles(settings.finalAssetBundlePath,
-                BuildAssetBundleOptions.UncompressedAssetBundle | BuildAssetBundleOptions.ForceRebuildAssetBundle,
+                BuildAssetBundleOptions.UncompressedAssetBundle | BuildAssetBundleOptions.ForceRebuildAssetBundle | BuildAssetBundleOptions.AssetBundleStripUnityVersion,
                 target);
 
             var afterSecondBuild = EditorApplication.timeSinceStartup;
@@ -1138,7 +1135,7 @@ void FinallyImportAsset()
         }
 
         private IGltfImport CreateGltfImport(AssetPath filePath) =>
-            env.gltfImporter.GetImporter(filePath, contentTable, settings.shaderType);
+            env.gltfImporter.GetImporter(filePath, contentTable, settings.shaderType, settings.buildTarget);
 
         private void ReduceTextureSizeIfNeeded(string texturePath, float maxSize)
         {

asset-bundle-converter/Assets/AssetBundleConverter/AssetBundleSceneConversionWindow.cs

@@ -40,6 +40,8 @@ public class AssetBundleSceneConversionWindow : EditorWindow
         private string entityId = "QmYy2TMDEfag99yZV4ZdpjievYUfdQgBVfFHKCDAge3zQi";
         private string wearablesCollectionId = "urn:decentraland:off-chain:base-avatars";
         private string debugEntity = "bafkreib66ufmbowp4ee2u3kdu6t52kouie7kd7tfrlv3l5kejz6yjcaq5i";
+
+        // The empty scenes url should be like "https://cdn.decentraland.org/@dcl/explorer/1.0.152291-20231017100112.commit-07d38e3/loader/empty-scenes/contents/"
         private string mappingName = "../mappings.json";
         private string batchBaseUrl = "";
         private string batchSceneId = "";

asset-bundle-converter/Assets/AssetBundleConverter/Editor/CustomGltfImporter.cs

@@ -73,7 +73,7 @@ public override void OnImportAsset(AssetImportContext ctx)
             }
 
             if (!useOriginalMaterials)
-                SetupCustomMaterialGenerator(new AssetBundleConverterMaterialGenerator());
+                SetupCustomMaterialGenerator(new AssetBundleConverterMaterialGenerator(AssetBundleConverterMaterialGenerator.UseNewShader(EditorUserBuildSettings.activeBuildTarget)));
 
             try { base.OnImportAsset(ctx); }
             catch (Exception e)

asset-bundle-converter/Assets/AssetBundleConverter/Tests/AssetBundleConverterShould.cs

@@ -149,7 +149,7 @@ public async Task GltfAssetIsProcessed()
             buildPipeline.BuildAssetBundles(Arg.Any<string>(), Arg.Any<BuildAssetBundleOptions>(), Arg.Any<BuildTarget>()).Returns(Substitute.For<IAssetBundleManifest>());
             var gltf = Substitute.For<IGltfImport>();
 
-            gltfImporter.GetImporter(Arg.Any<AssetPath>(), Arg.Any<Dictionary<string, string>>(), Arg.Any<ShaderType>()).Returns(gltf);
+            gltfImporter.GetImporter(Arg.Any<AssetPath>(), Arg.Any<Dictionary<string, string>>(), Arg.Any<ShaderType>(), Arg.Any<BuildTarget>()).Returns(gltf);
             gltfImporter.ConfigureImporter(Arg.Any<string>(), Arg.Any<ContentMap[]>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<ShaderType>()).Returns(true);
             assetDatabase.LoadAssetAtPath<GameObject>(PathUtils.FullPathToAssetPath(assetPath.finalPath)).Returns(dummyGo);
 
@@ -170,7 +170,7 @@ public async Task GltfAssetIsProcessed()
             file.Received().WriteAllBytes(Arg.Is(assetPath.finalPath), Arg.Any<byte[]>());
 
             // Ensure that the importer is being created
-            gltfImporter.Received().GetImporter(Arg.Any<AssetPath>(), Arg.Any<Dictionary<string, string>>(), Arg.Any<ShaderType>());
+            gltfImporter.Received().GetImporter(Arg.Any<AssetPath>(), Arg.Any<Dictionary<string, string>>(), Arg.Any<ShaderType>(), Arg.Any<BuildTarget>());
 
             // Ensure that the gltf import is being loaded
             await gltf.Received().Load(Arg.Any<string>(), Arg.Any<ImportSettings>());
@@ -275,7 +275,7 @@ private IGltfImport ConfigureGltf(ContentServerUtils.MappingPair mappingPair)
             webRequest.Get(exampleBaseURL).Returns(new DownloadHandlerMock());
             buildPipeline.BuildAssetBundles(Arg.Any<string>(), Arg.Any<BuildAssetBundleOptions>(), Arg.Any<BuildTarget>()).Returns(Substitute.For<IAssetBundleManifest>());
             var gltf = Substitute.For<IGltfImport>();
-            gltfImporter.GetImporter(Arg.Any<AssetPath>(), Arg.Any<Dictionary<string, string>>(), Arg.Any<ShaderType>()).Returns(gltf);
+            gltfImporter.GetImporter(Arg.Any<AssetPath>(), Arg.Any<Dictionary<string, string>>(), Arg.Any<ShaderType>(), Arg.Any<BuildTarget>()).Returns(gltf);
             gltfImporter.ConfigureImporter(Arg.Any<string>(), Arg.Any<ContentMap[]>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<ShaderType>()).Returns(true);
             assetDatabase.LoadAssetAtPath<GameObject>(PathUtils.FullPathToAssetPath(assetPath.finalPath)).Returns(dummyGo);
             return gltf;

asset-bundle-converter/Assets/AssetBundleConverter/Utils.cs

@@ -241,6 +241,25 @@ internal static void MarkFolderForAssetBundleBuild(string fullPath, string abNam
             importer.SetAssetBundleNameAndVariant(abName, "");
         }
 
+        internal static void AssignShaderBundle(IAssetDatabase db, Shader shader)
+        {
+            var abName = shader.name + "_IGNORE" + PlatformUtils.GetPlatform();
+
+            string assetPath = PathUtils.GetRelativePathTo(Application.dataPath, db.GetAssetPath(shader));
+
+            var importer = AssetImporter.GetAtPath(assetPath);
+
+            if (importer)
+                importer.SetAssetBundleNameAndVariant(abName, "");
+
+            // find a variants collection
+            var variantsPath = assetPath.Replace(".shader", "Variants.shadervariants");
+
+            importer = AssetImporter.GetAtPath(variantsPath);
+            if (importer)
+                importer.SetAssetBundleNameAndVariant(abName, "");
+        }
+
         internal static bool MarkAssetForAssetBundleBuild(IAssetDatabase assetDb, Object asset, string abName)
         {
             string assetPath = PathUtils.GetRelativePathTo(Application.dataPath, assetDb.GetAssetPath(asset));

asset-bundle-converter/Assets/AssetBundleConverter/Wrappers/Implementations/Default/AssetBundleConverterMaterialGenerator.cs

@@ -1,9 +1,40 @@
 using DCL.GLTFast.Wrappers;
+using DCL.Shaders;
+using GLTFast;
+using UnityEditor;
+using UnityEngine;
 
 namespace AssetBundleConverter.Wrappers.Implementations.Default
 {
     public class AssetBundleConverterMaterialGenerator : DecentralandMaterialGenerator
     {
-        public AssetBundleConverterMaterialGenerator() : base("DCL/Universal Render Pipeline/Lit") { }
+        private readonly bool useSceneShader;
+
+        public AssetBundleConverterMaterialGenerator(bool useSceneShader) : base(GetShaderName(useSceneShader))
+        {
+            this.useSceneShader = useSceneShader;
+        }
+
+        public static bool UseNewShader(BuildTarget buildTarget) =>
+            buildTarget != BuildTarget.WebGL;
+
+        private static string GetShaderName(bool useSceneShader) =>
+            useSceneShader ? "DCL/Scene" : "DCL/Universal Render Pipeline/Lit";
+
+        public override Material GenerateMaterial(int materialIndex, GLTFast.Schema.Material gltfMaterial, IGltfReadable gltf, bool pointsSupport = false)
+        {
+            var mat = base.GenerateMaterial(materialIndex, gltfMaterial, gltf, pointsSupport);
+
+            if (useSceneShader)
+            {
+                // Enable Forward+ and soft shadows
+                mat.EnableKeyword(ShaderUtils.FW_PLUS);
+                mat.EnableKeyword(ShaderUtils.FW_PLUS_LIGHT_SHADOWS);
+                mat.EnableKeyword(ShaderUtils.FW_PLUS_SHADOWS_CASCADE);
+                mat.EnableKeyword(ShaderUtils.FW_PLUS_SHADOWS_SOFT);
+            }
+
+            return mat;
+        }
     }
-}
+}

asset-bundle-converter/Assets/AssetBundleConverter/Wrappers/Implementations/Default/AssetDatabase.cs

@@ -82,9 +82,14 @@ public void CreateAsset(Object obj, string path)
                 UnityEditor.AssetDatabase.CreateAsset(obj, path);
             }
 
-            public void MarkAssetBundle(IAssetDatabase assetDb, Object asset, string abName)
+            public void AssignAssetBundle(Shader shader)
             {
-                Utils.MarkAssetForAssetBundleBuild(assetDb, asset, abName);
+                Utils.AssignShaderBundle(this, shader);
+            }
+
+            public void MarkAssetBundle(Object asset, string abName)
+            {
+                Utils.MarkAssetForAssetBundleBuild(this, asset, abName);
             }
         }
     }

asset-bundle-converter/Assets/AssetBundleConverter/Wrappers/Implementations/Default/DefaultGltfImporter.cs

@@ -1,15 +1,12 @@
-
-using AssetBundleConverter.Editor;
+using AssetBundleConverter.Editor;
 using AssetBundleConverter.Wrappers.Interfaces;
 using DCL;
 using DCL.ABConverter;
 using GLTFast;
 using GLTFast.Logging;
 using GLTFast.Materials;
 using System.Collections.Generic;
-using System.Linq;
 using UnityEditor;
-using UnityEngine;
 
 namespace AssetBundleConverter.Wrappers.Implementations.Default
 {
@@ -26,9 +23,9 @@ public DefaultGltfImporter(IAssetDatabase assetDatabase)
             this.assetDatabase = assetDatabase;
         }
 
-        public IGltfImport GetImporter(AssetPath filePath, Dictionary<string, string> contentTable, ShaderType shaderType)
+        public IGltfImport GetImporter(AssetPath filePath, Dictionary<string, string> contentTable, ShaderType shaderType, BuildTarget buildTarget)
         {
-            getNewMaterialGenerator = GetNewMaterialGenerator(shaderType);
+            getNewMaterialGenerator = GetNewMaterialGenerator(shaderType, buildTarget);
 
             return new GltfImportWrapper(
                 new GltFastFileProvider(filePath.fileRootPath, filePath.hash, contentTable),
@@ -37,13 +34,10 @@ public IGltfImport GetImporter(AssetPath filePath, Dictionary<string, string> co
                 gltfLogger);
         }
 
-        private IMaterialGenerator GetNewMaterialGenerator(ShaderType shaderType)
-        {
-            if (shaderType == ShaderType.Dcl)
-                return new AssetBundleConverterMaterialGenerator();
-
-            return null;
-        }
+        private static IMaterialGenerator GetNewMaterialGenerator(ShaderType shaderType, BuildTarget buildTarget) =>
+            shaderType == ShaderType.Dcl
+                ? new AssetBundleConverterMaterialGenerator(AssetBundleConverterMaterialGenerator.UseNewShader(buildTarget))
+                : null;
 
         public bool ConfigureImporter(string relativePath, ContentMap[] contentMap, string fileRootPath, string hash, ShaderType shaderType )
         {