Skip to content

Commit

Permalink
Add external path and type whitelist to ResourceLoader
Browse files Browse the repository at this point in the history 
Updated documentation for whitelisted resource loader.
  • Loading branch information
@fire
fire committed Dec 14, 2024
1 parent dc5f1b7 commit 2d81804
Show file tree
Hide file tree
Showing 9 changed files with 167 additions and 14 deletions.
14 changes: 14 additions & 0 deletions core/core_bind.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ Error ResourceLoader::load_threaded_request(const String &p_path, const String &
return ::ResourceLoader::load_threaded_request(p_path, p_type_hint, p_use_sub_threads, ResourceFormatLoader::CacheMode(p_cache_mode));
}

Error ResourceLoader::load_threaded_request_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_type_hint, bool p_use_sub_threads, CacheMode p_cache_mode) {
return ::ResourceLoader::load_threaded_request_whitelisted(p_path, p_external_path_whitelist, p_type_whitelist, p_type_hint, p_use_sub_threads, ResourceFormatLoader::CacheMode(p_cache_mode));
}

ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, Array r_progress) {
float progress = 0;
::ResourceLoader::ThreadLoadStatus tls = ::ResourceLoader::load_threaded_get_status(p_path, &progress);
Expand All @@ -79,6 +83,14 @@ Ref<Resource> ResourceLoader::load(const String &p_path, const String &p_type_hi
return ret;
}

Ref<Resource> ResourceLoader::load_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_type_hint, CacheMode p_cache_mode) {
Error err = OK;
Ref<Resource> ret = ::ResourceLoader::load_whitelisted(p_path, p_external_path_whitelist, p_type_whitelist, p_type_hint, ResourceFormatLoader::CacheMode(p_cache_mode), &err);

ERR_FAIL_COND_V_MSG(err != OK, ret, "Error loading resource: '" + p_path + "'.");
return ret;
}

Vector<String> ResourceLoader::get_recognized_extensions_for_type(const String &p_type) {
List<String> exts;
::ResourceLoader::get_recognized_extensions_for_type(p_type, &exts);
Expand Down Expand Up @@ -138,10 +150,12 @@ Vector<String> ResourceLoader::list_directory(const String &p_directory) {

void ResourceLoader::_bind_methods() {
ClassDB::bind_method(D_METHOD("load_threaded_request", "path", "type_hint", "use_sub_threads", "cache_mode"), &ResourceLoader::load_threaded_request, DEFVAL(""), DEFVAL(false), DEFVAL(CACHE_MODE_REUSE));
ClassDB::bind_method(D_METHOD("load_threaded_request_whitelisted", "path", "external_path_whitelist", "type_whitelist", "type_hint", "use_sub_threads", "cache_mode"), &ResourceLoader::load_threaded_request_whitelisted, DEFVAL(""), DEFVAL(false), DEFVAL(CACHE_MODE_REUSE));
ClassDB::bind_method(D_METHOD("load_threaded_get_status", "path", "progress"), &ResourceLoader::load_threaded_get_status, DEFVAL_ARRAY);
ClassDB::bind_method(D_METHOD("load_threaded_get", "path"), &ResourceLoader::load_threaded_get);

ClassDB::bind_method(D_METHOD("load", "path", "type_hint", "cache_mode"), &ResourceLoader::load, DEFVAL(""), DEFVAL(CACHE_MODE_REUSE));
ClassDB::bind_method(D_METHOD("load_whitelisted", "path", "external_path_whitelist", "type_whitelist", "type_hint", "cache_mode"), &ResourceLoader::load_whitelisted, DEFVAL(""), DEFVAL(CACHE_MODE_REUSE));
ClassDB::bind_method(D_METHOD("get_recognized_extensions_for_type", "type"), &ResourceLoader::get_recognized_extensions_for_type);
ClassDB::bind_method(D_METHOD("add_resource_format_loader", "format_loader", "at_front"), &ResourceLoader::add_resource_format_loader, DEFVAL(false));
ClassDB::bind_method(D_METHOD("remove_resource_format_loader", "format_loader"), &ResourceLoader::remove_resource_format_loader);
Expand Down
2 changes: 2 additions & 0 deletions core/core_bind.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,12 @@ class ResourceLoader : public Object {
static ResourceLoader *get_singleton() { return singleton; }

Error load_threaded_request(const String &p_path, const String &p_type_hint = "", bool p_use_sub_threads = false, CacheMode p_cache_mode = CACHE_MODE_REUSE);
Error load_threaded_request_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_type_hint = "", bool p_use_sub_threads = false, CacheMode p_cache_mode = CACHE_MODE_REUSE);
ThreadLoadStatus load_threaded_get_status(const String &p_path, Array r_progress = ClassDB::default_array_arg);
Ref<Resource> load_threaded_get(const String &p_path);

Ref<Resource> load(const String &p_path, const String &p_type_hint = "", CacheMode p_cache_mode = CACHE_MODE_REUSE);
Ref<Resource> load_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_type_hint = "", CacheMode p_cache_mode = CACHE_MODE_REUSE);
Vector<String> get_recognized_extensions_for_type(const String &p_type);
void add_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader, bool p_at_front);
void remove_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader);
Expand Down
55 changes: 51 additions & 4 deletions core/io/resource_format_binary.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,8 +432,10 @@ Error ResourceLoaderBinary::parse_variant(Variant &r_v) {
path = remaps[path];
}

Ref<Resource> res = ResourceLoader::load(path, exttype, cache_mode_for_external);

Ref<Resource> res;
if (!using_whitelist || external_path_whitelist.has(path)) {
res = ResourceLoader::load(path, exttype, cache_mode_for_external);
}
if (res.is_null()) {
WARN_PRINT(vformat("Couldn't load resource: %s.", path));
}
Expand Down Expand Up @@ -698,7 +700,14 @@ Error ResourceLoaderBinary::load() {
}

external_resources.write[i].path = path; //remap happens here, not on load because on load it can actually be used for filesystem dock resource remap
external_resources.write[i].load_token = ResourceLoader::_load_start(path, external_resources[i].type, use_sub_threads ? ResourceLoader::LOAD_THREAD_DISTRIBUTE : ResourceLoader::LOAD_THREAD_FROM_CURRENT, cache_mode_for_external);

if (using_whitelist && !external_path_whitelist.has(path)) {
error = ERR_FILE_MISSING_DEPENDENCIES;
ERR_FAIL_V_MSG(error, "External dependency not in whitelist: " + path + ".");
}

external_resources.write[i].load_token = ResourceLoader::_load_start(path, external_resources[i].type, use_sub_threads ? ResourceLoader::LOAD_THREAD_DISTRIBUTE : ResourceLoader::LOAD_THREAD_FROM_CURRENT, ResourceFormatLoader::CACHE_MODE_REUSE, false, false, Dictionary(), Dictionary());

if (!external_resources[i].load_token.is_valid()) {
if (!ResourceLoader::get_abort_on_missing_resources()) {
ResourceLoader::notify_dependency_error(local_path, path, external_resources[i].type);
Expand Down Expand Up @@ -770,7 +779,10 @@ Error ResourceLoaderBinary::load() {
if (res.is_null()) {
//did not replace

Object *obj = ClassDB::instantiate(t);
Object *obj = nullptr;
if (!using_whitelist || type_whitelist.has(t)) {
obj = ClassDB::instantiate(t);
}
if (!obj) {
if (ResourceLoader::is_creating_missing_resources_if_class_unavailable_enabled()) {
//create a missing resource
Expand Down Expand Up @@ -1248,6 +1260,41 @@ Ref<Resource> ResourceFormatLoaderBinary::load(const String &p_path, const Strin
String path = !p_original_path.is_empty() ? p_original_path : p_path;
loader.local_path = ProjectSettings::get_singleton()->localize_path(path);
loader.res_path = loader.local_path;
loader.using_whitelist = false;
loader.open(f);

err = loader.load();

if (r_error) {
*r_error = err;
}

if (err) {
return Ref<Resource>();
}
return loader.resource;
}

Ref<Resource> ResourceFormatLoaderBinary::load_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, CacheMode p_cache_mode) {
if (r_error) {
*r_error = ERR_FILE_CANT_OPEN;
}

Error err;
Ref<FileAccess> f = FileAccess::open(p_path, FileAccess::READ, &err);

ERR_FAIL_COND_V_MSG(err != OK, Ref<Resource>(), "Cannot open file '" + p_path + "'.");

ResourceLoaderBinary loader;
loader.cache_mode = p_cache_mode;
loader.use_sub_threads = p_use_sub_threads;
loader.progress = r_progress;
String path = !p_original_path.is_empty() ? p_original_path : p_path;
loader.local_path = ProjectSettings::get_singleton()->localize_path(path);
loader.res_path = loader.local_path;
loader.using_whitelist = true;
loader.external_path_whitelist = p_external_path_whitelist;
loader.type_whitelist = p_type_whitelist;
loader.open(f);

err = loader.load();
Expand Down
5 changes: 5 additions & 0 deletions core/io/resource_format_binary.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ class ResourceLoaderBinary {
HashMap<String, String> remaps;
Error error = OK;

bool using_whitelist = false;
Dictionary external_path_whitelist;
Dictionary type_whitelist;

ResourceFormatLoader::CacheMode cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE;
ResourceFormatLoader::CacheMode cache_mode_for_external = ResourceFormatLoader::CACHE_MODE_REUSE;

Expand Down Expand Up @@ -111,6 +115,7 @@ class ResourceLoaderBinary {
class ResourceFormatLoaderBinary : public ResourceFormatLoader {
public:
virtual Ref<Resource> load(const String &p_path, const String &p_original_path = "", Error *r_error = nullptr, bool p_use_sub_threads = false, float *r_progress = nullptr, CacheMode p_cache_mode = CACHE_MODE_REUSE) override;
virtual Ref<Resource> load_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_original_path = "", Error *r_error = nullptr, bool p_use_sub_threads = false, float *r_progress = nullptr, CacheMode p_cache_mode = CACHE_MODE_REUSE) override;
virtual void get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const override;
virtual void get_recognized_extensions(List<String> *p_extensions) const override;
virtual bool handles_type(const String &p_type) const override;
Expand Down
2 changes: 1 addition & 1 deletion core/io/resource_importer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ Ref<Resource> ResourceFormatImporter::load_internal(const String &p_path, Error
}
}

Ref<Resource> res = ResourceLoader::_load(pat.path, p_path, pat.type, p_cache_mode, r_error, p_use_sub_threads, r_progress);
Ref<Resource> res = ResourceLoader::_load(pat.path, p_path, pat.type, p_cache_mode, false, Dictionary(), Dictionary(), r_error, p_use_sub_threads, r_progress);

#ifdef TOOLS_ENABLED
if (res.is_valid()) {
Expand Down
60 changes: 54 additions & 6 deletions core/io/resource_loader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ void ResourceFormatLoader::_bind_methods() {
GDVIRTUAL_BIND(_exists, "path");
GDVIRTUAL_BIND(_get_classes_used, "path");
GDVIRTUAL_BIND(_load, "path", "original_path", "use_sub_threads", "cache_mode");
//GDVIRTUAL_BIND(_load_whitelisted, "path", "external_path_whitelist", "type_Whitelist", "original_path", "use_sub_threads", "cache_mode");
}

///////////////////////////////////
Expand Down Expand Up @@ -279,7 +280,7 @@ ResourceLoader::LoadToken::~LoadToken() {
clear();
}

Ref<Resource> ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, ResourceFormatLoader::CacheMode p_cache_mode, Error *r_error, bool p_use_sub_threads, float *r_progress) {
Ref<Resource> ResourceLoader::_load(const String &p_path, const String &p_original_path, const String &p_type_hint, ResourceFormatLoader::CacheMode p_cache_mode, bool p_using_whitelist, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, Error *r_error, bool p_use_sub_threads, float *r_progress) {
const String &original_path = p_original_path.is_empty() ? p_path : p_original_path;
load_nesting++;
if (load_paths_stack.size()) {
Expand All @@ -302,7 +303,11 @@ Ref<Resource> ResourceLoader::_load(const String &p_path, const String &p_origin
continue;
}
found = true;
res = loader[i]->load(p_path, original_path, r_error, p_use_sub_threads, r_progress, p_cache_mode);
if (p_using_whitelist) {
res = loader[i]->load_whitelisted(p_path, p_external_path_whitelist, p_type_whitelist, !p_original_path.is_empty() ? p_original_path : p_path, r_error, p_use_sub_threads, r_progress, p_cache_mode);
} else {
res = loader[i]->load(p_path, !p_original_path.is_empty() ? p_original_path : p_path, r_error, p_use_sub_threads, r_progress, p_cache_mode);
}
if (!res.is_null()) {
break;
}
Expand Down Expand Up @@ -364,7 +369,17 @@ void ResourceLoader::_run_load_task(void *p_userdata) {
print_verbose("Loading resource: " + remapped_path);

Error load_err = OK;
Ref<Resource> res = _load(remapped_path, remapped_path != load_task.local_path ? load_task.local_path : String(), load_task.type_hint, load_task.cache_mode, &load_err, load_task.use_sub_threads, &load_task.progress);
Ref<Resource> res = _load(remapped_path,
remapped_path != load_task.local_path ? load_task.local_path : String(),
load_task.type_hint,
load_task.cache_mode,
load_task.using_whitelist,
load_task.external_path_whitelist,
load_task.type_whitelist,
&load_err,
load_task.use_sub_threads,
&load_task.progress);

if (MessageQueue::get_singleton() != MessageQueue::get_main_singleton()) {
MessageQueue::get_singleton()->flush();
}
Expand Down Expand Up @@ -489,7 +504,12 @@ static String _validate_local_path(const String &p_path) {
}

Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, ResourceFormatLoader::CacheMode p_cache_mode) {
Ref<ResourceLoader::LoadToken> token = _load_start(p_path, p_type_hint, p_use_sub_threads ? LOAD_THREAD_DISTRIBUTE : LOAD_THREAD_SPAWN_SINGLE, p_cache_mode, true);
Ref<ResourceLoader::LoadToken> token = _load_start(p_path, p_type_hint, p_use_sub_threads ? LOAD_THREAD_DISTRIBUTE : LOAD_THREAD_SPAWN_SINGLE, p_cache_mode, true, false, Dictionary(), Dictionary());
return token.is_valid() ? OK : FAILED;
}

Error ResourceLoader::load_threaded_request_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_type_hint, bool p_use_sub_threads, ResourceFormatLoader::CacheMode p_cache_mode) {
Ref<ResourceLoader::LoadToken> token = _load_start(p_path, p_type_hint, p_use_sub_threads ? LOAD_THREAD_DISTRIBUTE : LOAD_THREAD_SPAWN_SINGLE, p_cache_mode, true, true, p_external_path_whitelist, p_type_whitelist);
return token.is_valid() ? OK : FAILED;
}

Expand Down Expand Up @@ -526,7 +546,32 @@ Ref<Resource> ResourceLoader::load(const String &p_path, const String &p_type_hi
// cyclic load detection and awaiting.
thread_mode = LOAD_THREAD_SPAWN_SINGLE;
}
Ref<LoadToken> load_token = _load_start(p_path, p_type_hint, thread_mode, p_cache_mode);
Ref<LoadToken> load_token = _load_start(p_path, p_type_hint, thread_mode, p_cache_mode, false, false, Dictionary(), Dictionary());
if (!load_token.is_valid()) {
if (r_error) {
*r_error = FAILED;
}
return Ref<Resource>();
}

Ref<Resource> res = _load_complete(*load_token.ptr(), r_error);
return res;
}

Ref<Resource> ResourceLoader::load_whitelisted(const String &p_path, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist, const String &p_type_hint, ResourceFormatLoader::CacheMode p_cache_mode, Error *r_error) {
if (r_error) {
*r_error = OK;
}

LoadThreadMode thread_mode = LOAD_THREAD_FROM_CURRENT;
if (WorkerThreadPool::get_singleton()->get_caller_task_id() != WorkerThreadPool::INVALID_TASK_ID) {
// If user is initiating a single-threaded load from a WorkerThreadPool task,
// we instead spawn a new task so there's a precondition that a load in a pool task
// is always initiated by the engine. That makes certain aspects simpler, such as
// cyclic load detection and awaiting.
thread_mode = LOAD_THREAD_SPAWN_SINGLE;
}
Ref<LoadToken> load_token = _load_start(p_path, p_type_hint, thread_mode, p_cache_mode, false, true, p_external_path_whitelist, p_type_whitelist);
if (!load_token.is_valid()) {
if (r_error) {
*r_error = FAILED;
Expand All @@ -538,7 +583,7 @@ Ref<Resource> ResourceLoader::load(const String &p_path, const String &p_type_hi
return res;
}

Ref<ResourceLoader::LoadToken> ResourceLoader::_load_start(const String &p_path, const String &p_type_hint, LoadThreadMode p_thread_mode, ResourceFormatLoader::CacheMode p_cache_mode, bool p_for_user) {
Ref<ResourceLoader::LoadToken> ResourceLoader::_load_start(const String &p_path, const String &p_type_hint, LoadThreadMode p_thread_mode, ResourceFormatLoader::CacheMode p_cache_mode, bool p_for_user, bool p_use_whitelist, Dictionary p_external_path_whitelist, Dictionary p_type_whitelist) {
String local_path = _validate_local_path(p_path);

bool ignoring_cache = p_cache_mode == ResourceFormatLoader::CACHE_MODE_IGNORE || p_cache_mode == ResourceFormatLoader::CACHE_MODE_IGNORE_DEEP;
Expand Down Expand Up @@ -587,6 +632,9 @@ Ref<ResourceLoader::LoadToken> ResourceLoader::_load_start(const String &p_path,
load_task.type_hint = p_type_hint;
load_task.cache_mode = p_cache_mode;
load_task.use_sub_threads = p_thread_mode == LOAD_THREAD_DISTRIBUTE;
load_task.using_whitelist = p_use_whitelist;
load_task.external_path_whitelist = p_external_path_whitelist;
load_task.type_whitelist = p_type_whitelist;
if (p_cache_mode == ResourceFormatLoader::CACHE_MODE_REUSE) {
Ref<Resource> existing = ResourceCache::get_ref(local_path);
if (existing.is_valid()) {
Expand Down
Loading

0 comments on commit 2d81804

Please sign in to comment.