refactor: improve content loading (#132)

* implement http requester with priority queue and only async
replace content mapping Dictionary by Arc<ContentMapping>
implement content provider in Rust

* gltf and texture loading working async

* wearables/avatar/backpack working async

* download video working

* refix auto animations gltfs

* temp remove video player test => events not working well

godot/assets/sky/sky_test.tscn

@@ -13,7 +13,6 @@ sky = SubResource("Sky_e2lcq")
 ambient_light_source = 2
 ambient_light_color = Color(1, 1, 1, 1)
 reflected_light_source = 1
-glow_enabled = true
 glow_levels/1 = 16.0
 glow_levels/2 = 16.0
 glow_levels/3 = 16.0

godot/src/decentraland_components/audio_source.gd

@@ -30,19 +30,17 @@ func _async_refresh_data():
 	if last_loaded_audio_clip == dcl_audio_clip_url:
 		apply_audio_props(true)
 	else:
-		var content_mapping = Global.scene_runner.get_scene_content_mapping(dcl_scene_id)
+		var content_mapping := Global.scene_runner.get_scene_content_mapping(dcl_scene_id)
 
 		last_loaded_audio_clip = dcl_audio_clip_url
 		valid = false
 
-		var audio_clip_file_hash = content_mapping.get("content", {}).get(
-			last_loaded_audio_clip, ""
-		)
+		var audio_clip_file_hash = content_mapping.get_hash(last_loaded_audio_clip)
 		if audio_clip_file_hash.is_empty():
 			# TODO: log file not found
 			return
 
-		var promise: Promise = Global.content_manager.fetch_audio(
+		var promise: Promise = Global.content_provider.fetch_audio(
 			last_loaded_audio_clip, content_mapping
 		)
 		var res = await PromiseUtils.async_awaiter(promise)

godot/src/decentraland_components/avatar.gd

@@ -97,15 +97,15 @@ func async_update_avatar(avatar: Dictionary):
 	current_skin_color = Avatar.from_color_object(avatar.get("skin", {}).get("color", null))
 	current_hair_color = Avatar.from_color_object(avatar.get("hair", {}).get("color", null))
 
-	var wearable_to_request := PackedStringArray(current_wearables)
+	var wearable_to_request := Array(current_wearables)
 	wearable_to_request.push_back(current_body_shape)
 
 	_load_default_emotes()
 
 	finish_loading = false
 
-	var promise = Global.content_manager.fetch_wearables(wearable_to_request, current_content_url)
-	await PromiseUtils.async_awaiter(promise)
+	var promise = Global.content_provider.fetch_wearables(wearable_to_request, current_content_url)
+	await PromiseUtils.async_all(promise)
 	async_fetch_wearables_dependencies()
 
 
@@ -201,9 +201,9 @@ func async_fetch_wearables_dependencies():
 	wearables_dict.clear()
 
 	# Fill data
-	wearables_dict[current_body_shape] = Global.content_manager.get_wearable(current_body_shape)
+	wearables_dict[current_body_shape] = Global.content_provider.get_wearable(current_body_shape)
 	for item in current_wearables:
-		wearables_dict[item] = Global.content_manager.get_wearable(item)
+		wearables_dict[item] = Global.content_provider.get_wearable(item)
 
 	var async_calls: Array = []
 	for wearable_key in wearables_dict.keys():
@@ -224,33 +224,28 @@ func async_fetch_wearables_dependencies():
 		if hashes_to_fetch.is_empty():
 			continue
 
-		var content: Dictionary = wearable.get("content", {})
-		var content_to_fetch := {}
-		for file_name in content:
+		var content_mapping: DclContentMappingAndUrl = wearable.get("content")
+		var files: Array = []
+		for file_name in content_mapping.get_files():
 			for file_hash in hashes_to_fetch:
-				if content[file_name] == file_hash:
-					content_to_fetch[file_name] = file_hash
-
-		var content_mapping: Dictionary = {
-			"content": wearable.get("content", {}),
-			"base_url": "https://peer.decentraland.org/content/contents/"
-		}
+				if content_mapping.get_hash(file_name) == file_hash:
+					files.push_back(file_name)
 
-		for file_name in content_to_fetch:
+		for file_name in files:
 			async_calls.push_back(_fetch_texture_or_gltf(file_name, content_mapping))
 
 	await PromiseUtils.async_all(async_calls)
 
 	async_load_wearables()
 
 
-func _fetch_texture_or_gltf(file_name, content_mapping):
+func _fetch_texture_or_gltf(file_name: String, content_mapping: DclContentMappingAndUrl) -> Promise:
 	var promise: Promise
 
 	if file_name.ends_with(".png"):
-		promise = Global.content_manager.fetch_texture(file_name, content_mapping)
+		promise = Global.content_provider.fetch_texture(file_name, content_mapping)
 	else:
-		promise = Global.content_manager.fetch_gltf(file_name, content_mapping)
+		promise = Global.content_provider.fetch_gltf(file_name, content_mapping)
 
 	return promise
 
@@ -267,7 +262,7 @@ func _free_old_skeleton(skeleton: Node):
 
 
 func try_to_set_body_shape(body_shape_hash):
-	var body_shape: Node3D = Global.content_manager.get_resource_from_hash(body_shape_hash)
+	var body_shape: Node3D = Global.content_provider.get_gltf_from_hash(body_shape_hash)
 	if body_shape == null:
 		return
 
@@ -331,7 +326,7 @@ func async_load_wearables():
 			continue
 
 		var file_hash = Wearables.get_wearable_main_file_hash(wearable, current_body_shape)
-		var obj = Global.content_manager.get_resource_from_hash(file_hash)
+		var obj = Global.content_provider.get_gltf_from_hash(file_hash)
 		var wearable_skeleton: Skeleton3D = obj.find_child("Skeleton3D")
 		for child in wearable_skeleton.get_children():
 			var new_wearable = child.duplicate()
@@ -376,7 +371,7 @@ func async_load_wearables():
 			child.mesh = child.mesh.duplicate(true)
 			meshes.push_back({"n": child.get_surface_override_material_count(), "mesh": child.mesh})
 
-	var promise = Global.content_manager.duplicate_materials(meshes)
+	var promise: Promise = Global.content_provider.duplicate_materials(meshes)
 	await PromiseUtils.async_awaiter(promise)
 	apply_color_and_facial()
 	body_shape_skeleton_3d.visible = true
@@ -439,14 +434,14 @@ func apply_texture_and_mask(
 ):
 	var current_material = mask_material.duplicate()
 	current_material.set_shader_parameter(
-		"base_texture", Global.content_manager.get_resource_from_hash(textures[0])
+		"base_texture", Global.content_provider.get_texture_from_hash(textures[0])
 	)
 	current_material.set_shader_parameter("material_color", color)
 	current_material.set_shader_parameter("mask_color", mask_color)
 
 	if textures.size() > 1:
 		current_material.set_shader_parameter(
-			"mask_texture", Global.content_manager.get_resource_from_hash(textures[1])
+			"mask_texture", Global.content_provider.get_texture_from_hash(textures[1])
 		)
 
 	mesh.mesh.surface_set_material(0, current_material)

godot/src/decentraland_components/gltf_container.gd

@@ -17,10 +17,10 @@ func _ready():
 
 
 func async_load_gltf():
-	var content_mapping = Global.scene_runner.get_scene_content_mapping(dcl_scene_id)
+	var content_mapping := Global.scene_runner.get_scene_content_mapping(dcl_scene_id)
 
 	self.dcl_gltf_src = dcl_gltf_src.to_lower()
-	self.file_hash = content_mapping.get("content", {}).get(dcl_gltf_src, "")
+	self.file_hash = content_mapping.get_hash(dcl_gltf_src)
 
 	if self.file_hash.is_empty():
 		dcl_gltf_loading_state = GltfContainerLoadingState.NOT_FOUND
@@ -29,20 +29,37 @@ func async_load_gltf():
 	# TODO: should we set a timeout?
 	dcl_gltf_loading_state = GltfContainerLoadingState.LOADING
 
-	var promise = Global.content_manager.fetch_gltf(dcl_gltf_src, content_mapping)
-	if promise != null:
+	var promise = Global.content_provider.fetch_gltf(dcl_gltf_src, content_mapping)
+	if promise == null:
+		printerr("Fatal error on fetch gltf: promise == null")
+		return
+
+	if not promise.is_resolved():
 		await PromiseUtils.async_awaiter(promise)
 
-	_async_on_gltf_loaded()
+	var res = promise.get_data()
+	if res is PromiseError:
+		printerr("Error on fetch gltf: ", res.get_error())
+		return
+
+	var instance_promise: Promise = Global.content_provider.instance_gltf_colliders(
+		res, dcl_visible_cmask, dcl_invisible_cmask, dcl_scene_id, dcl_entity_id
+	)
+	var res_instance = await PromiseUtils.async_awaiter(instance_promise)
+	if res_instance is PromiseError:
+		printerr("Error on fetch gltf: ", res_instance.get_error())
+		return
+
+	self.async_deferred_add_child.call_deferred(res_instance)
 
 
 func _async_on_gltf_loaded():
-	var node = Global.content_manager.get_resource_from_hash(file_hash)
+	var node = Global.content_provider.get_gltf_from_hash(file_hash)
 	if node == null:
 		dcl_gltf_loading_state = GltfContainerLoadingState.FINISHED_WITH_ERROR
 		return
 
-	var promise: Promise = Global.content_manager.instance_gltf_colliders(
+	var promise: Promise = Global.content_provider.instance_gltf_colliders(
 		node, dcl_visible_cmask, dcl_invisible_cmask, dcl_scene_id, dcl_entity_id
 	)
 
@@ -60,13 +77,14 @@ func async_deferred_add_child(new_gltf_node):
 		return
 
 	add_child(new_gltf_node)
-	self.check_animations()
 
 	await main_tree.process_frame
 
 	# Colliders and rendering is ensured to be ready at this point
 	dcl_gltf_loading_state = GltfContainerLoadingState.FINISHED
 
+	self.check_animations()
+
 
 func get_animatable_body_3d(mesh_instance: MeshInstance3D):
 	for maybe_static_body in mesh_instance.get_children():

godot/src/decentraland_components/video_player.gd

@@ -9,9 +9,9 @@ func stream_buffer(data: PackedVector2Array):
 
 
 func async_request_video(file_hash):
-	var content_mapping = Global.scene_runner.get_scene_content_mapping(dcl_scene_id)
+	var content_mapping := Global.scene_runner.get_scene_content_mapping(dcl_scene_id)
 
-	var promise = Global.content_manager.fetch_video(file_hash, content_mapping)
+	var promise = Global.content_provider.fetch_video(file_hash, content_mapping)
 	var res = await PromiseUtils.async_awaiter(promise)
 	if res is PromiseError:
 		printerr("Error on fetching video: ", res.get_error())

godot/src/decentraland_components/wearables/wearables.gd

@@ -517,7 +517,7 @@ static func compose_hidden_categories(
 
 
 static func get_skeleton_from_content(content_hash: String) -> Skeleton3D:
-	var content = Global.content_manager.get_resource_from_hash(content_hash)
+	var content = Global.content_provider.get_gltf_from_hash(content_hash)
 	if content == null:
 		return null
 
@@ -544,21 +544,22 @@ static func get_wearable_facial_hashes(wearable: Variant, body_shape_id: String)
 		return []
 
 	var main_file: String = representation.get("mainFile", "").to_lower()
-	var content = wearable.get("content", {})
-	var main_texture_file_hash = content.get(main_file, "")
+	var content_mapping: DclContentMappingAndUrl = wearable.get("content")
+	var files := content_mapping.get_files()
+	var main_texture_file_hash = content_mapping.get_hash(main_file)
 	if main_texture_file_hash.is_empty():
-		for file_name in content:
+		for file_name in files:
 			if file_name.ends_with(".png") and not file_name.ends_with("_mask.png"):
-				main_texture_file_hash = content[file_name]
+				main_texture_file_hash = content_mapping.get_hash(file_name)
 				break
 
 	if main_texture_file_hash.is_empty():
 		return []
 
 	var mask_texture_file_hash: String
-	for file_name in content:
+	for file_name in files:
 		if file_name.ends_with("_mask.png"):
-			mask_texture_file_hash = content[file_name]
+			mask_texture_file_hash = content_mapping.get_hash(file_name)
 			break
 
 	if mask_texture_file_hash.is_empty():
@@ -576,7 +577,8 @@ static func get_wearable_main_file_hash(wearable: Variant, body_shape_id: String
 		return ""
 
 	var main_file: String = representation.get("mainFile", "").to_lower()
-	var file_hash = wearable.get("content", {}).get(main_file, "")
+	var content_mapping: DclContentMappingAndUrl = wearable.get("content")
+	var file_hash = content_mapping.get_hash(main_file)
 	return file_hash
 
 
@@ -591,12 +593,15 @@ static func is_valid_wearable(
 		return false
 
 	var main_file: String = representation.get("mainFile", "").to_lower()
-	var file_hash = wearable.get("content", {}).get(main_file, "")
+	var content_mapping: DclContentMappingAndUrl = wearable.get("content")
+	var file_hash = content_mapping.get_hash(main_file)
 	if file_hash.is_empty():
 		return false
 
 	if not skip_content_integrity:
-		var obj = Global.content_manager.get_resource_from_hash(file_hash)
+		var obj = Global.content_provider.get_gltf_from_hash(file_hash)
+		if obj == null:
+			obj = Global.content_provider.get_texture_from_hash(file_hash)
 		if obj == null:
 			# printerr("wearable ", wearable_key, " doesn't have resource from hash")
 			return false
@@ -623,14 +628,14 @@ static func get_curated_wearable_list(
 ) -> Array:
 	var wearables_by_category: Dictionary = {}
 
-	var body_shape = Global.content_manager.get_wearable(body_shape_id)
+	var body_shape = Global.content_provider.get_wearable(body_shape_id)
 	if not is_valid_wearable(body_shape, body_shape_id):
 		return []
 
 	wearables_by_category[Categories.BODY_SHAPE] = body_shape
 
 	for wearable_id in wearables:
-		var wearable = Global.content_manager.get_wearable(wearable_id)
+		var wearable = Global.content_provider.get_wearable(wearable_id)
 		if is_valid_wearable(wearable, body_shape_id):
 			var category = get_category(wearable)
 			if not wearables_by_category.has(category):
@@ -667,9 +672,9 @@ static func set_fallback_for_missing_needed_categories(
 			needed_catagory
 		)
 		if fallback_wearable_id != null:
-			var fallback_wearable = Global.content_manager.get_wearable(fallback_wearable_id)
+			var fallback_wearable = Global.content_provider.get_wearable(fallback_wearable_id)
 			if is_valid_wearable(fallback_wearable, body_shape_id):
-				wearables_by_category[needed_catagory] = Global.content_manager.get_wearable(
+				wearables_by_category[needed_catagory] = Global.content_provider.get_wearable(
 					fallback_wearable_id
 				)
 

godot/src/global.gd

@@ -16,14 +16,13 @@ const FORCE_TEST_REALM = "https://decentraland.github.io/scene-explorer-tests/sc
 # const FORCE_TEST_REALM = "http://localhost:8000"
 
 ## Global classes (singleton pattern)
-var content_manager: ContentManager
 var config: ConfigData
 
 var raycast_debugger = load("res://src/tool/raycast_debugger/raycast_debugger.gd").new()
 var animation_importer: AnimationImporter = AnimationImporter.new()
 
 var scene_fetcher: SceneFetcher = null
-var http_requester: RustHttpRequesterWrapper
+var http_requester: RustHttpQueueRequester
 
 var nft_fetcher: OpenSeaFetcher = OpenSeaFetcher.new()
 var nft_frame_loader: NftFrameStyleLoader = NftFrameStyleLoader.new()
@@ -36,7 +35,7 @@ var dcl_android_plugin
 
 
 func _ready():
-	http_requester = RustHttpRequesterWrapper.new()
+	http_requester = RustHttpQueueRequester.new()
 
 	var args := OS.get_cmdline_args()
 	if args.size() == 1 and args[0].begins_with("res://"):
@@ -71,8 +70,8 @@ func _ready():
 	self.testing_tools = TestingTools.new()
 	self.testing_tools.set_name("testing_tool")
 
-	self.content_manager = ContentManager.new()
-	self.content_manager.set_name("content_manager")
+	self.content_provider = ContentProvider.new()
+	self.content_provider.set_name("content_provider")
 
 	self.scene_fetcher = SceneFetcher.new()
 	self.scene_fetcher.set_name("scene_fetcher")
@@ -84,7 +83,7 @@ func _ready():
 	self.avatars.set_name("avatar_scene")
 
 	get_tree().root.add_child.call_deferred(self.scene_fetcher)
-	get_tree().root.add_child.call_deferred(self.content_manager)
+	get_tree().root.add_child.call_deferred(self.content_provider)
 	get_tree().root.add_child.call_deferred(self.scene_runner)
 	get_tree().root.add_child.call_deferred(self.realm)
 	get_tree().root.add_child.call_deferred(self.player_identity)
@@ -93,6 +92,9 @@ func _ready():
 	get_tree().root.add_child.call_deferred(self.portable_experience_controller)
 	get_tree().root.add_child.call_deferred(self.testing_tools)
 
+	var custom_importer = load("res://src/logic/custom_gltf_importer.gd").new()
+	GLTFDocument.register_gltf_document_extension(custom_importer)
+
 	# TODO: enable raycast debugger
 	add_child(raycast_debugger)
 
@@ -116,7 +118,8 @@ func print_node_tree(node: Node, prefix = ""):
 
 
 func _process(_dt: float):
-	http_requester.poll()
+	pass
+	#http_requester.poll()
 
 
 func get_explorer():