Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement textual ext/subresource IDs. #50676

Merged
merged 1 commit into from
Jul 22, 2021
Merged

Implement textual ext/subresource IDs. #50676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 42 additions & 10 deletions core/io/resource.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include "core/core_string_names.h"
#include "core/io/file_access.h"
#include "core/io/resource_loader.h"
#include "core/math/math_funcs.h"
#include "core/object/script_language.h"
#include "core/os/os.h"
#include "scene/main/node.h" //only so casting works
Expand Down Expand Up @@ -94,12 +95,43 @@ String Resource::get_path() const {
return path_cache;
}

void Resource::set_subindex(int p_sub_index) {
subindex = p_sub_index;
String Resource::generate_scene_unique_id() {
// Generate a unique enough hash, but still user-readable.
// If it's not unique it does not matter because the saver will try again.
OS::Date date = OS::get_singleton()->get_date();
OS::Time time = OS::get_singleton()->get_time();
uint32_t hash = hash_djb2_one_32(OS::get_singleton()->get_ticks_usec());
hash = hash_djb2_one_32(date.year, hash);
hash = hash_djb2_one_32(date.month, hash);
hash = hash_djb2_one_32(date.day, hash);
hash = hash_djb2_one_32(time.hour, hash);
hash = hash_djb2_one_32(time.minute, hash);
hash = hash_djb2_one_32(time.second, hash);
hash = hash_djb2_one_32(Math::rand(), hash);

static constexpr uint32_t characters = 5;
static constexpr uint32_t char_count = ('z' - 'a');
static constexpr uint32_t base = char_count + ('9' - '0');
String id;
for (uint32_t i = 0; i < characters; i++) {
uint32_t c = hash % base;
if (c < char_count) {
id += String::chr('a' + c);
} else {
id += String::chr('0' + (c - char_count));
}
hash /= base;
}

return id;
}

void Resource::set_scene_unique_id(const String &p_id) {
scene_unique_id = p_id;
}

int Resource::get_subindex() const {
return subindex;
String Resource::get_scene_unique_id() const {
return scene_unique_id;
}

void Resource::set_name(const String &p_name) {
Expand Down Expand Up @@ -350,8 +382,8 @@ bool Resource::is_translation_remapped() const {

#ifdef TOOLS_ENABLED
//helps keep IDs same number when loading/saving scenes. -1 clears ID and it Returns -1 when no id stored
void Resource::set_id_for_path(const String &p_path, int p_id) {
if (p_id == -1) {
void Resource::set_id_for_path(const String &p_path, const String &p_id) {
if (p_id == "") {
ResourceCache::path_cache_lock.write_lock();
ResourceCache::resource_path_cache[p_path].erase(get_path());
ResourceCache::path_cache_lock.write_unlock();
Expand All @@ -362,15 +394,15 @@ void Resource::set_id_for_path(const String &p_path, int p_id) {
}
}

int Resource::get_id_for_path(const String &p_path) const {
String Resource::get_id_for_path(const String &p_path) const {
ResourceCache::path_cache_lock.read_lock();
if (ResourceCache::resource_path_cache[p_path].has(get_path())) {
int result = ResourceCache::resource_path_cache[p_path][get_path()];
String result = ResourceCache::resource_path_cache[p_path][get_path()];
ResourceCache::path_cache_lock.read_unlock();
return result;
} else {
ResourceCache::path_cache_lock.read_unlock();
return -1;
return "";
}
}
#endif
Expand Down Expand Up @@ -414,7 +446,7 @@ Resource::~Resource() {

HashMap<String, Resource *> ResourceCache::resources;
#ifdef TOOLS_ENABLED
HashMap<String, HashMap<String, int>> ResourceCache::resource_path_cache;
HashMap<String, HashMap<String, String>> ResourceCache::resource_path_cache;
#endif

RWLock ResourceCache::lock;
Expand Down
13 changes: 7 additions & 6 deletions core/io/resource.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ class Resource : public RefCounted {

String name;
String path_cache;
int subindex = 0;
String scene_unique_id;

virtual bool _use_builtin_script() const { return true; }

Expand Down Expand Up @@ -105,8 +105,9 @@ class Resource : public RefCounted {
virtual void set_path(const String &p_path, bool p_take_over = false);
String get_path() const;

void set_subindex(int p_sub_index);
int get_subindex() const;
static String generate_scene_unique_id();
void set_scene_unique_id(const String &p_id);
String get_scene_unique_id() const;

virtual Ref<Resource> duplicate(bool p_subresources = false) const;
Ref<Resource> duplicate_for_local_scene(Node *p_for_scene, Map<Ref<Resource>, Ref<Resource>> &remap_cache);
Expand Down Expand Up @@ -140,8 +141,8 @@ class Resource : public RefCounted {

#ifdef TOOLS_ENABLED
//helps keep IDs same number when loading/saving scenes. -1 clears ID and it Returns -1 when no id stored
void set_id_for_path(const String &p_path, int p_id);
int get_id_for_path(const String &p_path) const;
void set_id_for_path(const String &p_path, const String &p_id);
String get_id_for_path(const String &p_path) const;
#endif

Resource();
Expand All @@ -156,7 +157,7 @@ class ResourceCache {
static RWLock lock;
static HashMap<String, Resource *> resources;
#ifdef TOOLS_ENABLED
static HashMap<String, HashMap<String, int>> resource_path_cache; // each tscn has a set of resource paths and IDs
static HashMap<String, HashMap<String, String>> resource_path_cache; // Each tscn has a set of resource paths and IDs.
static RWLock path_cache_lock;
#endif // TOOLS_ENABLED
friend void unregister_core_types();
Expand Down
86 changes: 51 additions & 35 deletions core/io/resource_format_binary.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,10 @@ enum {
OBJECT_EXTERNAL_RESOURCE = 1,
OBJECT_INTERNAL_RESOURCE = 2,
OBJECT_EXTERNAL_RESOURCE_INDEX = 3,
//version 2: added 64 bits support for float and int
//version 3: changed nodepath encoding
FORMAT_VERSION = 3,
// Version 2: added 64 bits support for float and int.
// Version 3: changed nodepath encoding.
// Version 4: new string ID for ext/subresources, breaks forward compat.
FORMAT_VERSION = 4,
FORMAT_VERSION_CAN_RENAME_DEPS = 1,
FORMAT_VERSION_NO_NODEPATH_PROPERTY = 3,
};
Expand Down Expand Up @@ -311,7 +312,14 @@ Error ResourceLoaderBinary::parse_variant(Variant &r_v) {
} break;
case OBJECT_INTERNAL_RESOURCE: {
uint32_t index = f->get_32();
String path = res_path + "::" + itos(index);
String path;

if (using_named_scene_ids) { // New format.
ERR_FAIL_INDEX_V((int)index, internal_resources.size(), ERR_PARSE_ERROR);
path = internal_resources[index].path;
} else {
path += res_path + "::" + itos(index);
}

//always use internal cache for loading internal resources
if (!internal_index_cache.has(path)) {
Expand All @@ -320,7 +328,6 @@ Error ResourceLoaderBinary::parse_variant(Variant &r_v) {
} else {
r_v = internal_index_cache[path];
}

} break;
case OBJECT_EXTERNAL_RESOURCE: {
//old file format, still around for compatibility
Expand Down Expand Up @@ -378,7 +385,6 @@ Error ResourceLoaderBinary::parse_variant(Variant &r_v) {
ERR_FAIL_V(ERR_FILE_CORRUPT);
} break;
}

} break;
case VARIANT_CALLABLE: {
r_v = Callable();
Expand Down Expand Up @@ -659,15 +665,17 @@ Error ResourceLoaderBinary::load() {

//maybe it is loaded already
String path;
int subindex = 0;
String id;

if (!main) {
path = internal_resources[i].path;

if (path.begins_with("local://")) {
path = path.replace_first("local://", "");
subindex = path.to_int();
id = path;
path = res_path + "::" + path;

internal_resources.write[i].path = path; // Update path.
}

if (cache_mode == ResourceFormatLoader::CACHE_MODE_REUSE) {
Expand Down Expand Up @@ -722,7 +730,7 @@ Error ResourceLoaderBinary::load() {
if (path != String() && cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE) {
r->set_path(path, cache_mode == ResourceFormatLoader::CACHE_MODE_REPLACE); //if got here because the resource with same path has different type, replace it
}
r->set_subindex(subindex);
r->set_scene_unique_id(id);
}

if (!main) {
Expand Down Expand Up @@ -879,7 +887,11 @@ void ResourceLoaderBinary::open(FileAccess *p_f) {
print_bl("type: " + type);

importmd_ofs = f->get_64();
for (int i = 0; i < 14; i++) {
uint32_t flags = f->get_32();
if (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_NAMED_SCENE_IDS) {
using_named_scene_ids = true;
}
for (int i = 0; i < 13; i++) {
f->get_32(); //skip a few reserved fields
}

Expand Down Expand Up @@ -1269,11 +1281,7 @@ void ResourceFormatSaverBinaryInstance::_pad_buffer(FileAccess *f, int p_bytes)
}
}

void ResourceFormatSaverBinaryInstance::_write_variant(const Variant &p_property, const PropertyInfo &p_hint) {
write_variant(f, p_property, resource_set, external_resources, string_map, p_hint);
}

void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Variant &p_property, Set<RES> &resource_set, Map<RES, int> &external_resources, Map<StringName, int> &string_map, const PropertyInfo &p_hint) {
void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Variant &p_property, Map<RES, int> &resource_map, Map<RES, int> &external_resources, Map<StringName, int> &string_map, const PropertyInfo &p_hint) {
switch (p_property.get_type()) {
case Variant::NIL: {
f->store_32(VARIANT_NIL);
Expand Down Expand Up @@ -1492,13 +1500,13 @@ void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Varia
f->store_32(OBJECT_EXTERNAL_RESOURCE_INDEX);
f->store_32(external_resources[res]);
} else {
if (!resource_set.has(res)) {
if (!resource_map.has(res)) {
f->store_32(OBJECT_EMPTY);
ERR_FAIL_MSG("Resource was not pre cached for the resource section, most likely due to circular reference.");
}

f->store_32(OBJECT_INTERNAL_RESOURCE);
f->store_32(res->get_subindex());
f->store_32(resource_map[res]);
//internal resource
}

Expand Down Expand Up @@ -1526,8 +1534,8 @@ void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Varia
continue;
*/

write_variant(f, E->get(), resource_set, external_resources, string_map);
write_variant(f, d[E->get()], resource_set, external_resources, string_map);
write_variant(f, E->get(), resource_map, external_resources, string_map);
write_variant(f, d[E->get()], resource_map, external_resources, string_map);
}

} break;
Expand All @@ -1536,7 +1544,7 @@ void ResourceFormatSaverBinaryInstance::write_variant(FileAccess *f, const Varia
Array a = p_property;
f->store_32(uint32_t(a.size()));
for (int i = 0; i < a.size(); i++) {
write_variant(f, a[i], resource_set, external_resources, string_map);
write_variant(f, a[i], resource_map, external_resources, string_map);
}

} break;
Expand Down Expand Up @@ -1816,7 +1824,8 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p

save_unicode_string(f, p_resource->get_class());
f->store_64(0); //offset to import metadata
for (int i = 0; i < 14; i++) {
f->store_32(FORMAT_FLAG_NAMED_SCENE_IDS);
for (int i = 0; i < 13; i++) {
f->store_32(0); // reserved
}

Expand Down Expand Up @@ -1886,37 +1895,43 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p
// save internal resource table
f->store_32(saved_resources.size()); //amount of internal resources
Vector<uint64_t> ofs_pos;
Set<int> used_indices;
Set<String> used_unique_ids;

for (List<RES>::Element *E = saved_resources.front(); E; E = E->next()) {
RES r = E->get();
if (r->get_path() == "" || r->get_path().find("::") != -1) {
if (r->get_subindex() != 0) {
if (used_indices.has(r->get_subindex())) {
r->set_subindex(0); //repeated
if (r->get_scene_unique_id() != "") {
if (used_unique_ids.has(r->get_scene_unique_id())) {
r->set_scene_unique_id("");
} else {
used_indices.insert(r->get_subindex());
used_unique_ids.insert(r->get_scene_unique_id());
}
}
}
}

Map<RES, int> resource_map;
int res_index = 0;
for (List<RES>::Element *E = saved_resources.front(); E; E = E->next()) {
RES r = E->get();
if (r->get_path() == "" || r->get_path().find("::") != -1) {
if (r->get_subindex() == 0) {
int new_subindex = 1;
if (used_indices.size()) {
new_subindex = used_indices.back()->get() + 1;
if (r->get_scene_unique_id() == "") {
String new_id;

while (true) {
new_id = r->get_class() + "_" + Resource::generate_scene_unique_id();
if (!used_unique_ids.has(new_id)) {
break;
}
}

r->set_subindex(new_subindex);
used_indices.insert(new_subindex);
r->set_scene_unique_id(new_id);
used_unique_ids.insert(new_id);
}

save_unicode_string(f, "local://" + itos(r->get_subindex()));
save_unicode_string(f, "local://" + r->get_scene_unique_id());
if (takeover_paths) {
r->set_path(p_path + "::" + itos(r->get_subindex()), true);
r->set_path(p_path + "::" + r->get_scene_unique_id(), true);
}
#ifdef TOOLS_ENABLED
r->set_edited(false);
Expand All @@ -1926,6 +1941,7 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p
}
ofs_pos.push_back(f->get_position());
f->store_64(0); //offset in 64 bits
resource_map[r] = res_index++;
}

Vector<uint64_t> ofs_table;
Expand All @@ -1941,7 +1957,7 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p
for (List<Property>::Element *F = rd.properties.front(); F; F = F->next()) {
Property &p = F->get();
f->store_32(p.name_idx);
_write_variant(p.value, F->get().pi);
write_variant(f, p.value, resource_map, external_resources, string_map, F->get().pi);
}
}

Expand Down
7 changes: 5 additions & 2 deletions core/io/resource_format_binary.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class ResourceLoaderBinary {
RES cache;
};

bool using_named_scene_ids = false;
bool use_sub_threads = false;
float *progress = nullptr;
Vector<ExtResource> external_resources;
Expand Down Expand Up @@ -150,14 +151,16 @@ class ResourceFormatSaverBinaryInstance {
};

static void _pad_buffer(FileAccess *f, int p_bytes);
void _write_variant(const Variant &p_property, const PropertyInfo &p_hint = PropertyInfo());
void _find_resources(const Variant &p_variant, bool p_main = false);
static void save_unicode_string(FileAccess *f, const String &p_string, bool p_bit_on_len = false);
int get_string_index(const String &p_string);

public:
enum {
FORMAT_FLAG_NAMED_SCENE_IDS = 1
};
Error save(const String &p_path, const RES &p_resource, uint32_t p_flags = 0);
static void write_variant(FileAccess *f, const Variant &p_property, Set<RES> &resource_set, Map<RES, int> &external_resources, Map<StringName, int> &string_map, const PropertyInfo &p_hint = PropertyInfo());
static void write_variant(FileAccess *f, const Variant &p_property, Map<RES, int> &resource_map, Map<RES, int> &external_resources, Map<StringName, int> &string_map, const PropertyInfo &p_hint = PropertyInfo());
};

class ResourceFormatSaverBinary : public ResourceFormatSaver {
Expand Down
Loading