Skip to content

Commit

Permalink
Importer version system for files imported in older Godot versions
Browse files Browse the repository at this point in the history
  • Loading branch information
aaronfranke committed Oct 31, 2023
1 parent 6afd320 commit 20d1201
Show file tree
Hide file tree
Showing 10 changed files with 123 additions and 65 deletions.
8 changes: 8 additions & 0 deletions core/io/resource_importer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,14 @@ ResourceFormatImporter::ResourceFormatImporter() {

//////////////

int ResourceImporter::get_importer_version() const {
return _importer_version;
}

void ResourceImporter::set_importer_version(int p_version) {
_importer_version = p_version;
}

void ResourceImporter::_bind_methods() {
BIND_ENUM_CONSTANT(IMPORT_ORDER_DEFAULT);
BIND_ENUM_CONSTANT(IMPORT_ORDER_SCENE);
Expand Down
7 changes: 6 additions & 1 deletion core/io/resource_importer.h
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ class ResourceFormatImporter : public ResourceFormatLoader {
class ResourceImporter : public RefCounted {
GDCLASS(ResourceImporter, RefCounted);

int _importer_version = get_latest_importer_version();

protected:
static void _bind_methods();

Expand All @@ -110,7 +112,10 @@ class ResourceImporter : public RefCounted {
virtual String get_resource_type() const = 0;
virtual float get_priority() const { return 1.0; }
virtual int get_import_order() const { return IMPORT_ORDER_DEFAULT; }
virtual int get_format_version() const { return 0; }
virtual int get_latest_importer_version(const String &p_file_extension = "") const { return 0; }

int get_importer_version() const;
void set_importer_version(int p_version);

struct ImportOption {
PropertyInfo option;
Expand Down
66 changes: 33 additions & 33 deletions editor/editor_file_system.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,6 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
String source_md5 = "";
Vector<String> dest_files;
String dest_md5 = "";
int version = 0;
bool found_uid = false;

while (true) {
Expand All @@ -418,8 +417,6 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
for (int i = 0; i < fa.size(); i++) {
to_check.push_back(fa[i]);
}
} else if (assign == "importer_version") {
version = value;
} else if (assign == "importer") {
importer_name = value;
} else if (assign == "uid") {
Expand Down Expand Up @@ -451,10 +448,6 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
return true; // the importer has possibly changed, try to reimport.
}

if (importer->get_format_version() > version) {
return true; // version changed, reimport
}

// Read the md5's from a separate file (so the import parameters aren't dependent on the file version
String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(p_path);
Ref<FileAccess> md5s = FileAccess::open(base_path + ".md5", FileAccess::READ, &err);
Expand Down Expand Up @@ -1820,10 +1813,7 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector
f->store_line("[remap]");
f->store_line("");
f->store_line("importer=\"" + importer->get_importer_name() + "\"");
int version = importer->get_format_version();
if (version > 0) {
f->store_line("importer_version=" + itos(version));
}
f->store_line("importer_version=" + itos(importer->get_importer_version()));
if (!importer->get_resource_type().is_empty()) {
f->store_line("type=\"" + importer->get_resource_type() + "\"");
}
Expand Down Expand Up @@ -1946,6 +1936,9 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
}

ResourceUID::ID uid = ResourceUID::INVALID_ID;
// When a .import file exists but it has no importer version written, use version 0.
// This is because older Godot versions did not write the version if it was 0.
int importer_version = 0;
Variant generator_parameters;
if (p_generator_parameters) {
generator_parameters = *p_generator_parameters;
Expand All @@ -1971,6 +1964,7 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
if (p_custom_importer.is_empty()) {
importer_name = cf->get_value("remap", "importer");
}
importer_version = cf->get_value("remap", "importer_version", 0);

if (cf->has_section_key("remap", "uid")) {
String uidt = cf->get_value("remap", "uid");
Expand All @@ -1984,6 +1978,10 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
}
}
}
} else {
// When resources do not have a .import file, use the latest importer
// version (to be determined below after getting the importer).
importer_version = -1;
}

if (importer_name == "keep") {
Expand All @@ -2003,17 +2001,22 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name);
}

const String file_extension = p_file.get_extension().to_lower();
if (importer.is_null()) {
//not found by name, find by extension
importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(p_file.get_extension());
importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(file_extension);
load_default = true;
if (importer.is_null()) {
ERR_FAIL_V_MSG(ERR_FILE_CANT_OPEN, "BUG: File queued for import, but can't be imported, importer for type '" + importer_name + "' not found.");
}
}

if (importer_version == -1) {
// When resources do not have a .import file, use the latest importer version.
importer_version = importer->get_latest_importer_version(file_extension);
}
importer->set_importer_version(importer_version);
//mix with default params, in case a parameter is missing

List<ResourceImporter::ImportOption> opts;
importer->get_import_options(p_file, &opts);
for (const ResourceImporter::ImportOption &E : opts) {
Expand Down Expand Up @@ -2054,10 +2057,7 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
f->store_line("[remap]");
f->store_line("");
f->store_line("importer=\"" + importer->get_importer_name() + "\"");
int version = importer->get_format_version();
if (version > 0) {
f->store_line("importer_version=" + itos(version));
}
f->store_line("importer_version=" + itos(importer->get_importer_version()));
if (!importer->get_resource_type().is_empty()) {
f->store_line("type=\"" + importer->get_resource_type() + "\"");
}
Expand Down Expand Up @@ -2217,7 +2217,7 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {

EditorProgress pr("reimport", TTR("(Re)Importing Assets"), p_files.size());

Vector<ImportFile> reimport_files;
Vector<ImportFile> files_to_reimport;

HashSet<String> groups_to_reimport;

Expand Down Expand Up @@ -2248,7 +2248,7 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
ifile.path = file;
ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(file, ifile.order, ifile.threaded, ifile.importer);
reloads.push_back(file);
reimport_files.push_back(ifile);
files_to_reimport.push_back(ifile);
}

// Group may have changed, so also update group reference.
Expand All @@ -2259,39 +2259,39 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
}
}

reimport_files.sort();
files_to_reimport.sort();

bool use_multiple_threads = GLOBAL_GET("editor/import/use_multiple_threads");

int from = 0;
for (int i = 0; i < reimport_files.size(); i++) {
if (groups_to_reimport.has(reimport_files[i].path)) {
for (int i = 0; i < files_to_reimport.size(); i++) {
if (groups_to_reimport.has(files_to_reimport[i].path)) {
continue;
}

if (use_multiple_threads && reimport_files[i].threaded) {
if (i + 1 == reimport_files.size() || reimport_files[i + 1].importer != reimport_files[from].importer) {
if (use_multiple_threads && files_to_reimport[i].threaded) {
if (i + 1 == files_to_reimport.size() || files_to_reimport[i + 1].importer != files_to_reimport[from].importer) {
if (from - i == 0) {
// Single file, do not use threads.
pr.step(reimport_files[i].path.get_file(), i);
_reimport_file(reimport_files[i].path);
pr.step(files_to_reimport[i].path.get_file(), i);
_reimport_file(files_to_reimport[i].path);
} else {
Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(reimport_files[from].importer);
Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(files_to_reimport[from].importer);
ERR_CONTINUE(!importer.is_valid());

importer->import_threaded_begin();

ImportThreadData tdata;
tdata.max_index = from;
tdata.reimport_from = from;
tdata.reimport_files = reimport_files.ptr();
tdata.reimport_files = files_to_reimport.ptr();

WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_singleton()->add_template_group_task(this, &EditorFileSystem::_reimport_thread, &tdata, i - from + 1, -1, false, vformat(TTR("Import resources of type: %s"), reimport_files[from].importer));
WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_singleton()->add_template_group_task(this, &EditorFileSystem::_reimport_thread, &tdata, i - from + 1, -1, false, vformat(TTR("Import resources of type: %s"), files_to_reimport[from].importer));
int current_index = from - 1;
do {
if (current_index < tdata.max_index) {
current_index = tdata.max_index;
pr.step(reimport_files[current_index].path.get_file(), current_index);
pr.step(files_to_reimport[current_index].path.get_file(), current_index);
}
OS::get_singleton()->delay_usec(1);
} while (!WorkerThreadPool::get_singleton()->is_group_task_completed(group_task));
Expand All @@ -2305,14 +2305,14 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
}

} else {
pr.step(reimport_files[i].path.get_file(), i);
_reimport_file(reimport_files[i].path);
pr.step(files_to_reimport[i].path.get_file(), i);
_reimport_file(files_to_reimport[i].path);
}
}

// Reimport groups.

from = reimport_files.size();
from = files_to_reimport.size();

if (groups_to_reimport.size()) {
HashMap<String, Vector<String>> group_files;
Expand Down
2 changes: 1 addition & 1 deletion editor/import/resource_importer_obj.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ String ResourceImporterOBJ::get_resource_type() const {
return "Mesh";
}

int ResourceImporterOBJ::get_format_version() const {
int ResourceImporterOBJ::get_latest_importer_version(const String &p_file_extension) const {
return 1;
}

Expand Down
2 changes: 1 addition & 1 deletion editor/import/resource_importer_obj.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class ResourceImporterOBJ : public ResourceImporter {
virtual void get_recognized_extensions(List<String> *p_extensions) const override;
virtual String get_save_extension() const override;
virtual String get_resource_type() const override;
virtual int get_format_version() const override;
virtual int get_latest_importer_version(const String &p_file_extension = "") const override;

virtual int get_preset_count() const override;
virtual String get_preset_name(int p_idx) const override;
Expand Down
54 changes: 32 additions & 22 deletions editor/import/resource_importer_scene.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,14 @@ Variant EditorSceneFormatImporter::get_option_visibility(const String &p_path, b
return ret;
}

int EditorSceneFormatImporter::get_importer_version() const {
return _importer_version;
}

void EditorSceneFormatImporter::set_importer_version(int p_version) {
_importer_version = p_version;
}

void EditorSceneFormatImporter::_bind_methods() {
GDVIRTUAL_BIND(_get_import_flags);
GDVIRTUAL_BIND(_get_extensions);
Expand Down Expand Up @@ -263,7 +271,11 @@ String ResourceImporterScene::get_resource_type() const {
return animation_importer ? "AnimationLibrary" : "PackedScene";
}

int ResourceImporterScene::get_format_version() const {
int ResourceImporterScene::get_latest_importer_version(const String &p_file_extension) const {
Ref<EditorSceneFormatImporter> scene_importer = get_scene_importer_for_file_extension(p_file_extension);
if (scene_importer.is_valid()) {
return scene_importer->get_importer_version();
}
return 1;
}

Expand Down Expand Up @@ -2390,29 +2402,14 @@ Node *ResourceImporterScene::pre_import(const String &p_source_file, const HashM
Error ResourceImporterScene::import(const String &p_source_file, const String &p_save_path, const HashMap<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files, Variant *r_metadata) {
const String &src_path = p_source_file;

Ref<EditorSceneFormatImporter> importer;
String ext = src_path.get_extension().to_lower();
Ref<EditorSceneFormatImporter> importer = get_scene_importer_for_file_extension(ext);

EditorProgress progress("import", TTR("Import Scene"), 104);
progress.step(TTR("Importing Scene..."), 0);

for (Ref<EditorSceneFormatImporter> importer_elem : importers) {
List<String> extensions;
importer_elem->get_extensions(&extensions);

for (const String &F : extensions) {
if (F.to_lower() == ext) {
importer = importer_elem;
break;
}
}

if (importer.is_valid()) {
break;
}
}

ERR_FAIL_COND_V(!importer.is_valid(), ERR_FILE_UNRECOGNIZED);
importer->set_importer_version(get_importer_version());

int import_flags = 0;

Expand Down Expand Up @@ -2700,6 +2697,23 @@ void ResourceImporterScene::add_importer(Ref<EditorSceneFormatImporter> p_import
}
}

void ResourceImporterScene::remove_importer(Ref<EditorSceneFormatImporter> p_importer) {
importers.erase(p_importer);
}

Ref<EditorSceneFormatImporter> ResourceImporterScene::get_scene_importer_for_file_extension(const String &p_file_extension) {
for (Ref<EditorSceneFormatImporter> scene_importer : importers) {
List<String> extensions;
scene_importer->get_extensions(&extensions);
for (const String &F : extensions) {
if (F.to_lower() == p_file_extension) {
return scene_importer;
}
}
}
return Ref<EditorSceneFormatImporter>();
}

void ResourceImporterScene::remove_post_importer_plugin(const Ref<EditorScenePostImportPlugin> &p_plugin) {
post_importer_plugins.erase(p_plugin);
}
Expand All @@ -2713,10 +2727,6 @@ void ResourceImporterScene::add_post_importer_plugin(const Ref<EditorScenePostIm
}
}

void ResourceImporterScene::remove_importer(Ref<EditorSceneFormatImporter> p_importer) {
importers.erase(p_importer);
}

void ResourceImporterScene::clean_up_importer_plugins() {
importers.clear();
post_importer_plugins.clear();
Expand Down
8 changes: 7 additions & 1 deletion editor/import/resource_importer_scene.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ class ImporterMesh;
class EditorSceneFormatImporter : public RefCounted {
GDCLASS(EditorSceneFormatImporter, RefCounted);

int _importer_version = 1;

protected:
static void _bind_methods();

Expand Down Expand Up @@ -80,6 +82,9 @@ class EditorSceneFormatImporter : public RefCounted {
virtual void get_import_options(const String &p_path, List<ResourceImporter::ImportOption> *r_options);
virtual Variant get_option_visibility(const String &p_path, bool p_for_animation, const String &p_option, const HashMap<StringName, Variant> &p_options);

int get_importer_version() const;
void set_importer_version(int p_version);

EditorSceneFormatImporter() {}
};

Expand Down Expand Up @@ -246,6 +251,7 @@ class ResourceImporterScene : public ResourceImporter {

static void add_importer(Ref<EditorSceneFormatImporter> p_importer, bool p_first_priority = false);
static void remove_importer(Ref<EditorSceneFormatImporter> p_importer);
static Ref<EditorSceneFormatImporter> get_scene_importer_for_file_extension(const String &p_file_extension);

static void clean_up_importer_plugins();

Expand All @@ -254,7 +260,7 @@ class ResourceImporterScene : public ResourceImporter {
virtual void get_recognized_extensions(List<String> *p_extensions) const override;
virtual String get_save_extension() const override;
virtual String get_resource_type() const override;
virtual int get_format_version() const override;
virtual int get_latest_importer_version(const String &p_file_extension = "") const override;

virtual int get_preset_count() const override;
virtual String get_preset_name(int p_idx) const override;
Expand Down
1 change: 1 addition & 0 deletions modules/gltf/editor/editor_scene_importer_gltf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ Node *EditorSceneFormatImporterGLTF::import_scene(const String &p_path, uint32_t
gltf.instantiate();
Ref<GLTFState> state;
state.instantiate();
gltf->set_importer_version(get_importer_version());
if (p_options.has("gltf/embedded_image_handling")) {
int32_t enum_option = p_options["gltf/embedded_image_handling"];
state->set_handle_binary_image(enum_option);
Expand Down
Loading

0 comments on commit 20d1201

Please sign in to comment.