Skip to content
This repository has been archived by the owner on Feb 20, 2023. It is now read-only.

Browsing history and site data causing excessive storage bloat #26420

Closed
KennethHeal opened this issue Aug 10, 2022 · 63 comments
Closed

Browsing history and site data causing excessive storage bloat #26420

KennethHeal opened this issue Aug 10, 2022 · 63 comments
Labels
performance Possible performance wins qa-triaged Issues triaged by qa S2 Major Functionality/product severely impaired and a satisfactory workaround doesn't exist

Comments

@KennethHeal
Copy link

KennethHeal commented Aug 10, 2022

Steps to reproduce

None in particular, just use fenix.

Expected behavior

Storage space used to remain stable and clearing browser cached images and files to yield some savings

Actual behavior

Storage space used keeps bloating and emptying browser cached images and files is useless.
Deleting browser history and site data clears majority of the bloat, but results in loss of history.

Device information


About:support output

{
"application": {
"name": "Firefox",
"osVersion": "Linux 25 #1 SMP PREEMPT Wed Nov 15 16:06:34 KST 2017",
"version": "103.0.2",
"buildID": "20220802163236",
"distributionID": "",
"userAgent": "Mozilla/5.0 (Android 7.1.1; Mobile; rv:103.0) Gecko/103.0 Firefox/103.0",
"safeMode": false,
"memorySizeBytes": 1497272320,
"diskAvailableBytes": 467644416,
"supportURL": "https://support.mozilla.org/1/mobile/103.0.2/Android/en-US/",
"osTheme": "",
"numTotalWindows": 0,
"numFissionWindows": 0,
"numRemoteWindows": 0,
"fissionAutoStart": false,
"fissionDecisionStatus": "disabledByDefault",
"remoteAutoStart": true,
"keyLocationServiceGoogleFound": true,
"keySafebrowsingGoogleFound": true,
"keyMozillaFound": true
},
"securitySoftware": {
"registeredAntiVirus": "",
"registeredAntiSpyware": "",
"registeredFirewall": ""
},
"processes": {
"remoteTypes": {
"web": 1,
"gpu": 1
},
"maxWebContentProcesses": 2
},
"environmentVariables": {},
"modifiedPreferences": {
"browser.search.region": "NL",
"extensions.lastAppVersion": "103.0.2",
"idle.lastDailyNotification": 1660082742,
"media.gmp.storage.version.observed": 1,
"network.dns.disablePrefetch": true,
"network.http.speculative-parallel-limit": 0,
"network.predictor.enabled": false,
"network.prefetch-next": false,
"privacy.purge_trackers.last_purge": "1660082755461",
"privacy.purge_trackers.date_in_cookie_database": "0",
"security.remote_settings.crlite_filters.checked": 1656588114,
"security.remote_settings.intermediates.checked": 1656588114,
"storage.vacuum.last.index": 0
},
"lockedPreferences": {
"fission.autostart.session": false
},
"printingPreferences": {},
"media": {
"currentAudioBackend": "opensl",
"currentMaxAudioChannels": 2,
"currentPreferredSampleRate": 48000,
"audioOutputDevices": [
{
"name": "Default audio output device",
"groupId": "",
"vendor": "",
"type": 2,
"state": 2,
"preferred": 15,
"supportedFormat": 12336,
"defaultFormat": 16,
"maxChannels": 2,
"defaultRate": 44100,
"maxRate": 44100,
"minRate": 44100,
"maxLatency": 441,
"minLatency": 128
}
],
"audioInputDevices": [
{
"name": "Default audio input device",
"groupId": "",
"vendor": "",
"type": 1,
"state": 2,
"preferred": 15,
"supportedFormat": 12336,
"defaultFormat": 16,
"maxChannels": 1,
"defaultRate": 44100,
"maxRate": 44100,
"minRate": 44100,
"maxLatency": 441,
"minLatency": 128
}
]
},
"accessibility": {
"isActive": true,
"forceDisabled": 0,
"handlerUsed": false,
"instantiator": ""
},
"startupCache": {
"DiskCachePath": "/data/user/0/org.mozilla.firefox/files/mozilla/ft7e13yv.default/startupCache/startupCache.4.little",
"IgnoreDiskCache": false,
"FoundDiskCacheOnInit": false,
"WroteToDiskCache": true
},
"libraryVersions": {
"NSPR": {
"minVersion": "4.34",
"version": "4.34"
},
"NSS": {
"minVersion": "3.80",
"version": "3.80"
},
"NSSUTIL": {
"minVersion": "3.80",
"version": "3.80"
},
"NSSSSL": {
"minVersion": "3.80",
"version": "3.80"
},
"NSSSMIME": {
"minVersion": "3.80",
"version": "3.80"
}
},
"userJS": {
"exists": false
},
"intl": {
"localeService": {
"requested": [
"en-NL",
"en-GB"
],
"available": [
"an",
"ar",
"ast",
"az",
"be",
"bg",
"bn",
"br",
"bs",
"ca",
"cak",
"cs",
"cy",
"da",
"de",
"dsb",
"el",
"en-CA",
"en-GB",
"en-US",
"eo",
"es-AR",
"es-CL",
"es-ES",
"es-MX",
"et",
"eu",
"fa",
"ff",
"fi",
"fr",
"fy-NL",
"ga-IE",
"gd",
"gl",
"gn",
"gu-IN",
"he",
"hi-IN",
"hr",
"hsb",
"hu",
"hy-AM",
"id",
"is",
"it",
"ja",
"ka",
"kab",
"kk",
"kn",
"ko",
"lij",
"lo",
"lt",
"lv",
"ml",
"mr",
"ms",
"my",
"nb-NO",
"ne-NP",
"nl",
"nn-NO",
"oc",
"pa-IN",
"pl",
"pt-BR",
"pt-PT",
"rm",
"ro",
"ru",
"sk",
"sl",
"son",
"sq",
"sr",
"sv-SE",
"ta",
"te",
"th",
"tr",
"trs",
"uk",
"ur",
"uz",
"vi",
"wo",
"xh",
"zam",
"zh-CN",
"zh-TW"
],
"supported": [
"en-US",
"en-CA",
"en-GB"
],
"regionalPrefs": [
"en-NL",
"en-GB"
],
"defaultLocale": "en-US"
},
"osPrefs": {
"systemLocales": [
"en-NL",
"en-GB"
],
"regionalPrefsLocales": [
"en-NL",
"en-GB"
]
}
},
"crashes": {
"submitted": [],
"pending": 0
},
"remoteAgent": {
"running": false,
"url": ""
},
"graphics": {
"numTotalWindows": 2,
"numAcceleratedWindows": 2,
"windowLayerManagerType": "WebRender",
"windowLayerManagerRemote": true,
"adapterDescription": "Model: SM-T560NU, Product: gtelwifiue, Manufacturer: samsung, Hardware: qcom, OpenGL: Qualcomm -- Adreno (TM) 306 -- OpenGL ES 3.0 V@145.0 AU@ (GIT@I750f7f2fa6)",
"adapterVendorID": "Qualcomm",
"adapterDeviceID": "Adreno (TM) 306",
"adapterRAM": 0,
"adapterDrivers": "",
"driverVendor": "",
"driverVersion": "OpenGL ES 3.0 V@145.0 AU@ (GIT@I750f7f2fa6)",
"driverDate": "",
"targetFrameRate": 60,
"webgl1Renderer": "Qualcomm -- Adreno (TM) 306",
"webgl1Version": "OpenGL ES 3.0 V@145.0 AU@ (GIT@I750f7f2fa6)",
"webgl1DriverExtensions": "GL_AMD_compressed_ATC_texture GL_AMD_performance_monitor GL_AMD_program_binary_Z400 GL_EXT_debug_label GL_EXT_debug_marker GL_EXT_discard_framebuffer GL_EXT_robustness GL_EXT_texture_format_BGRA8888 GL_EXT_texture_type_2_10_10_10_REV GL_NV_fence GL_OES_compressed_ETC1_RGB8_texture GL_OES_depth_texture GL_OES_depth24 GL_OES_EGL_image GL_OES_EGL_sync GL_OES_EGL_image_external GL_OES_element_index_uint GL_OES_fbo_render_mipmap GL_OES_fragment_precision_high GL_OES_get_program_binary GL_OES_packed_depth_stencil GL_OES_depth_texture_cube_map GL_OES_rgb8_rgba8 GL_OES_standard_derivatives GL_OES_texture_3D GL_OES_texture_float GL_OES_texture_half_float GL_OES_texture_half_float_linear GL_OES_texture_npot GL_OES_vertex_half_float GL_OES_vertex_type_10_10_10_2 GL_OES_vertex_array_object GL_QCOM_alpha_test GL_QCOM_binning_control GL_QCOM_driver_control GL_QCOM_perfmon_global_mode GL_QCOM_extended_get GL_QCOM_extended_get2 GL_QCOM_tiled_rendering GL_QCOM_writeonly_rendering GL_EXT_sRGB GL_EXT_sRGB_write_control GL_EXT_texture_sRGB_decode GL_EXT_texture_filter_anisotropic GL_EXT_multisampled_render_to_texture GL_EXT_color_buffer_float GL_EXT_color_buffer_half_float GL_EXT_disjoint_timer_query ",
"webgl1Extensions": "ANGLE_instanced_arrays EXT_blend_minmax EXT_color_buffer_half_float EXT_sRGB EXT_texture_filter_anisotropic MOZ_debug OES_element_index_uint OES_fbo_render_mipmap OES_standard_derivatives OES_texture_float OES_texture_half_float OES_texture_half_float_linear OES_vertex_array_object WEBGL_color_buffer_float WEBGL_compressed_texture_etc WEBGL_compressed_texture_etc1 WEBGL_debug_renderer_info WEBGL_debug_shaders WEBGL_depth_texture WEBGL_lose_context",
"webgl1WSIInfo": "EGL_VENDOR: Qualcomm, Inc\nEGL_VERSION: 1.4\nEGL_EXTENSIONS: EGL_QUALCOMM_shared_image EGL_KHR_image EGL_KHR_image_base EGL_QCOM_create_image EGL_KHR_lock_surface EGL_KHR_lock_surface2 EGL_KHR_lock_surface3 EGL_KHR_fence_sync EGL_KHR_cl_event EGL_KHR_cl_event2 EGL_IMG_context_priority EGL_KHR_gl_texture_2D_image EGL_KHR_gl_texture_cubemap_image EGL_KHR_gl_texture_3D_image EGL_KHR_gl_renderbuffer_image EGL_EXT_create_context_robustness EGL_ANDROID_blob_cache EGL_ANDROID_framebuffer_target EGL_KHR_create_context EGL_KHR_wait_sync EGL_KHR_gl_colorspace EGL_ANDROIDX_no_config_context EGL_KHR_partial_update EGL_ANDROID_image_crop EGL_ANDROID_recordable EGL_ANDROID_native_fence_sync EGL_ANDROID_image_native_buffer EGL_KHR_gl_colorspace EGL_IMG_image_plane_attribs \nIsWebglOutOfProcessEnabled: 0",
"webgl2Renderer": "Qualcomm -- Adreno (TM) 306",
"webgl2Version": "OpenGL ES 3.0 V@145.0 AU@ (GIT@I750f7f2fa6)",
"webgl2DriverExtensions": "GL_AMD_compressed_ATC_texture GL_AMD_performance_monitor GL_AMD_program_binary_Z400 GL_EXT_debug_label GL_EXT_debug_marker GL_EXT_discard_framebuffer GL_EXT_robustness GL_EXT_texture_format_BGRA8888 GL_EXT_texture_type_2_10_10_10_REV GL_NV_fence GL_OES_compressed_ETC1_RGB8_texture GL_OES_depth_texture GL_OES_depth24 GL_OES_EGL_image GL_OES_EGL_sync GL_OES_EGL_image_external GL_OES_element_index_uint GL_OES_fbo_render_mipmap GL_OES_fragment_precision_high GL_OES_get_program_binary GL_OES_packed_depth_stencil GL_OES_depth_texture_cube_map GL_OES_rgb8_rgba8 GL_OES_standard_derivatives GL_OES_texture_3D GL_OES_texture_float GL_OES_texture_half_float GL_OES_texture_half_float_linear GL_OES_texture_npot GL_OES_vertex_half_float GL_OES_vertex_type_10_10_10_2 GL_OES_vertex_array_object GL_QCOM_alpha_test GL_QCOM_binning_control GL_QCOM_driver_control GL_QCOM_perfmon_global_mode GL_QCOM_extended_get GL_QCOM_extended_get2 GL_QCOM_tiled_rendering GL_QCOM_writeonly_rendering GL_EXT_sRGB GL_EXT_sRGB_write_control GL_EXT_texture_sRGB_decode GL_EXT_texture_filter_anisotropic GL_EXT_multisampled_render_to_texture GL_EXT_color_buffer_float GL_EXT_color_buffer_half_float GL_EXT_disjoint_timer_query ",
"webgl2Extensions": "EXT_color_buffer_float EXT_texture_filter_anisotropic MOZ_debug WEBGL_compressed_texture_etc WEBGL_compressed_texture_etc1 WEBGL_debug_renderer_info WEBGL_debug_shaders WEBGL_lose_context",
"webgl2WSIInfo": "EGL_VENDOR: Qualcomm, Inc\nEGL_VERSION: 1.4\nEGL_EXTENSIONS: EGL_QUALCOMM_shared_image EGL_KHR_image EGL_KHR_image_base EGL_QCOM_create_image EGL_KHR_lock_surface EGL_KHR_lock_surface2 EGL_KHR_lock_surface3 EGL_KHR_fence_sync EGL_KHR_cl_event EGL_KHR_cl_event2 EGL_IMG_context_priority EGL_KHR_gl_texture_2D_image EGL_KHR_gl_texture_cubemap_image EGL_KHR_gl_texture_3D_image EGL_KHR_gl_renderbuffer_image EGL_EXT_create_context_robustness EGL_ANDROID_blob_cache EGL_ANDROID_framebuffer_target EGL_KHR_create_context EGL_KHR_wait_sync EGL_KHR_gl_colorspace EGL_ANDROIDX_no_config_context EGL_KHR_partial_update EGL_ANDROID_image_crop EGL_ANDROID_recordable EGL_ANDROID_native_fence_sync EGL_ANDROID_image_native_buffer EGL_KHR_gl_colorspace EGL_IMG_image_plane_attribs \nIsWebglOutOfProcessEnabled: 0",
"info": {
"AzureCanvasBackend (UI Process)": "skia",
"AzureFallbackCanvasBackend (UI Process)": "skia",
"AzureContentBackend (UI Process)": "skia",
"AzureCanvasBackend": "skia",
"AzureContentBackend": "skia",
"ApzTouchInput": 1,
"ApzAutoscrollInput": 1,
"ApzZoomingInput": 1,
"CMSOutputProfile": "Empty profile data",
"DisplayCount": 1,
"Display0": "800x1280@0Hz scales:1.000000|1.000000"
},
"failures": [
"CP+[GFX1-]: Receive IPC close with reason=AbnormalShutdown"
],
"indices": [
0
],
"featureLog": {
"features": [
{
"name": "HW_COMPOSITING",
"description": "Compositing",
"status": "force_enabled",
"log": [
{
"type": "default",
"status": "available"
},
{
"type": "user",
"status": "force_enabled",
"message": "Force-enabled by pref"
}
]
},
{
"name": "OPENGL_COMPOSITING",
"description": "OpenGL Compositing",
"status": "force_enabled",
"log": [
{
"type": "default",
"status": "available"
},
{
"type": "user",
"status": "force_enabled",
"message": "Force-enabled by pref"
}
]
},
{
"name": "GPU_PROCESS",
"description": "GPU Process",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
},
{
"name": "WEBRENDER",
"description": "WebRender",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
},
{
"name": "WEBRENDER_QUALIFIED",
"description": "WebRender qualified",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
},
{
"name": "WEBRENDER_COMPOSITOR",
"description": "WebRender native compositor",
"status": "disabled",
"log": [
{
"type": "default",
"status": "disabled",
"message": "Disabled by default"
}
]
},
{
"name": "WEBRENDER_PARTIAL",
"description": "WebRender partial present",
"status": "blocklisted",
"log": [
{
"type": "default",
"status": "available"
},
{
"type": "env",
"status": "blocklisted",
"message": "Blocklisted by gfxInfo"
}
]
},
{
"name": "WEBRENDER_SHADER_CACHE",
"description": "WebRender shader disk cache",
"status": "blocklisted",
"log": [
{
"type": "default",
"status": "available"
},
{
"type": "env",
"status": "blocklisted",
"message": "Blocklisted by gfxInfo"
}
]
},
{
"name": "WEBRENDER_OPTIMIZED_SHADERS",
"description": "WebRender optimized shaders",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
},
{
"name": "WEBRENDER_ANGLE",
"description": "WebRender ANGLE",
"status": "unavailable",
"log": [
{
"type": "default",
"status": "available"
},
{
"type": "env",
"status": "unavailable",
"message": "OS not supported"
}
]
},
{
"name": "WEBRENDER_DCOMP_PRESENT",
"description": "WebRender DirectComposition",
"status": "unavailable",
"log": [
{
"type": "default",
"status": "available"
},
{
"type": "user",
"status": "disabled",
"message": "User disabled via pref"
},
{
"type": "env",
"status": "unavailable",
"message": "Requires Windows 10 or later"
},
{
"type": "runtime",
"status": "unavailable",
"message": "Requires ANGLE"
}
]
},
{
"name": "WEBRENDER_SOFTWARE",
"description": "WebRender software fallback",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
},
{
"name": "WEBGPU",
"description": "WebGPU",
"status": "blocked",
"log": [
{
"type": "default",
"status": "disabled",
"message": "Disabled by default"
},
{
"type": "runtime",
"status": "blocked",
"message": "WebGPU cannot be enabled in release or beta"
}
]
},
{
"name": "VP8_HW_DECODE",
"description": "VP8 hardware decoding",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
},
{
"name": "VP9_HW_DECODE",
"description": "VP9 hardware decoding",
"status": "available",
"log": [
{
"type": "default",
"status": "available"
}
]
}
],
"fallbacks": []
},
"crashGuards": []
},
"addons": [
{
"name": "Mozilla Android Components - Ads Telemetry",
"type": "extension",
"version": "103.0.10.0803084751",
"isActive": true,
"id": "ads@mozac.org"
},
{
"name": "Mozilla Android Components - Browser Icons",
"type": "extension",
"version": "103.0.10.0803084737",
"isActive": true,
"id": "icons@mozac.org"
},
{
"name": "Mozilla Android Components - Firefox Accounts WebChannel",
"type": "extension",
"version": "103.0.10.0803084742",
"isActive": true,
"id": "fxa@mozac.org"
},
{
"name": "Mozilla Android Components - ReaderView",
"type": "extension",
"version": "103.0.10.0803084737",
"isActive": true,
"id": "readerview@mozac.org"
},
{
"name": "Mozilla Android Components - Search Telemetry",
"type": "extension",
"version": "103.0.10.0803084751",
"isActive": true,
"id": "cookies@mozac.org"
},
{
"name": "Mozilla Android Components - Web Compatibility Interventions",
"type": "extension",
"version": "103.0.0",
"isActive": true,
"id": "webcompat@mozilla.org"
},
{
"name": "uBlock Origin",
"type": "extension",
"version": "1.43.0",
"isActive": true,
"id": "uBlock0@raymondhill.net"
}
],
"features": []
}

  • Android device: SM-T560NU running 7.1.1
  • Fenix version: 103.0.2

┆Issue is synchronized with this Jira Task

@KennethHeal KennethHeal added the performance Possible performance wins label Aug 10, 2022
@github-actions github-actions bot added the needs:triage Issue needs triage label Aug 10, 2022
@csadilek csadilek removed the needs:triage Issue needs triage label Aug 10, 2022
@amedyne amedyne added the eng:qa:needed QA Needed label Aug 10, 2022
@amedyne
Copy link
Contributor

amedyne commented Aug 10, 2022

@sv-ohorvath Can the team help validate this issue and see if clear cache and clear image are working properly? Thanks!

@KennethHeal
Copy link
Author

KennethHeal commented Aug 11, 2022

Please let me know if I get any other useful data, force sqlite vacuum and/or browse filesystem of fenix data directory. Thanks.

@delia-pop
Copy link

We were not able to reproduce the issue so far on RC 103.2.0. The user data is reduced after deleting cached images and files data.

Devices used:

  • Tab Sony Xperia (Android 6.0.1)
  • LG G7 fit (Android 8.1.0)
  • OnePlus Nord N10 (Android 11)
  • Lenovo Yoga Tab 11 (Android 11)
  • Xiaomi 12 Pro (Android 12)

@delia-pop delia-pop added S2 Major Functionality/product severely impaired and a satisfactory workaround doesn't exist qa-triaged Issues triaged by qa and removed eng:qa:needed QA Needed labels Aug 11, 2022
@Dunexus
Copy link

Dunexus commented Aug 11, 2022

I am able to reproduce the high amount of user data stored.
This is an install of Fenix that I use daily and keep up to date for several years (since the beginning of fenix github releases).

@KennethHeal
Copy link
Author

Thanks for the updates. In my case for a couple of weeks usage hovered around 150mb but in the last week ballooned to the 236mb you see here.
Is there a way to browse the filesystem and/or check sqlite db sizes to establish what is taking up all that space?

@KennethHeal
Copy link
Author

KennethHeal commented Aug 15, 2022

Storage is still bloating

@KennethHeal KennethHeal changed the title Excessive storage space usage Browsing and site history causing excessive storage spa Aug 15, 2022
@KennethHeal KennethHeal changed the title Browsing and site history causing excessive storage spa Browsing and site history causing excessive storage bloat Aug 15, 2022
@KennethHeal
Copy link
Author

KennethHeal commented Aug 15, 2022

I conducted the following experiment of deleting just browsing history and site data.
At the start of this this was the space usage:

So I went ahead and deleted browsing history and site data.

This took a few minutes. Having done so Fenix showed this

And storage space usage

@KennethHeal KennethHeal changed the title Browsing and site history causing excessive storage bloat Browsing history and site data causing excessive storage bloat Aug 15, 2022
@KennethHeal
Copy link
Author

Unfortunately history failed to sync back so I lost all my history.... 😡

@KennethHeal
Copy link
Author

KennethHeal commented Aug 15, 2022

Btw 101mb is also bloated for just ublock addon, cookies and logins

@KennethHeal
Copy link
Author

KennethHeal commented Aug 27, 2022

It's still bloating away. At this stage it is near unusable.

@amedyne
Copy link
Contributor

amedyne commented Aug 28, 2022

@KennethHeal thank you for providing updates and additional info on this ticket.

@mcarare Can you help investigate further what is going on here? Thanks!

@mcarare
Copy link
Contributor

mcarare commented Aug 29, 2022

@KennethHeal Thank you for the investigations so far!

First, I would like to state that it is normal up to a point that upon visiting multiple web pages the "user data" (or simply "data" depending on OS) will increase continuously. That is caused by specific data to every website that is saved in order to optimize future visits to that page.

In your specific case, and that makes it harder for us to reproduce and investigate, it might be aggravated by specific pages you are visiting (as deleting site data decreases the space usage).

One thing that you could do that would help us investigate further, is to observe the "data" category size between navigating to pages that you visit regularly. If you notice a spike after opening one specific page that would be a piece of information that could help us reproduce and better diagnose the underlying problem.

Thank you!

@KennethHeal
Copy link
Author

@mcarare thanks for the follow up. I'd agree that some usage would be normal, but this bloating is beyond that. Unfortunately this bug caused me to lose all site history. I've not been able to detect much of a pattern, other than for the first week it seems okay and then the bloat sets in.
Unfortunately Fenix is a bit of a black box as there doesn't seem to be anyway to check the file system (no file:// or about:config), sqlite database size or where the data usage is coming from. Or am I missing any instrumentation or debug feature?

@mcarare
Copy link
Contributor

mcarare commented Aug 29, 2022

@KennethHeal If you have access in your file explorer app, you can check files in data/data/org.mozilla.firefox ... folders to see if either one stands out in terms of size.

@KennethHeal
Copy link
Author

@mcarare as my device is not rooted I don't see any of the files within my files app. Previously with file:// one could browse them.
Screenshot_20220829-142426

@KennethHeal
Copy link
Author

A couple days of light usage and its bloated to 230mb
Screenshot_20220830-200453

@KennethHeal
Copy link
Author

KennethHeal commented Sep 1, 2022

Bloaty McBloatfox... (updated to 104.1)
Screenshot_20220901-212026

@KennethHeal
Copy link
Author

KennethHeal commented Sep 2, 2022

As the bloat got out of hand I cleared all data, logged back into Firefox and resynced. Then went ahead and added uBlock Origin and logged into websites I commonly used. Having used it for half a day this is the usage.
Screenshot_20220902-170016
This is therefore a reasonable baseline. In a good functional scenario I would expect incremental growth from here.
It's a pity there's no debugging tools and no progress yet, so I may be forced to switch browser.

@KennethHeal
Copy link
Author

Fwiw, current Bloatfox release 106.1 stats
Screenshot_20221015-115652

@dveditz
Copy link

dveditz commented Oct 16, 2022

The one called simply "Firefox" is the release version, but the play store also has "Firefox Beta" and "Firefox Nightly" that you could run without needing a rooted phone. I'm not saying you should or that it would fix your problem, just explaining why I needed to confirm you were using a "Release" (capital R) version rather than one of the two developer versions. about:config is only available in the "Nightly" version, and I don't believe there's any other way to enable CRLite in Firefox for android.

I was wondering if we were running an experiment that enabled CRLite for a small population, but I only see a future one, and that will exclude mobile users.

@dveditz
Copy link

dveditz commented Oct 16, 2022

Unfortunately nobody has given me any pointers to pinpoint the issue I'm seeing, devs seem neither capable nor willing to figure out what space is being used by what! Fenix seems to lack any instrumentation whatsoever and file:// browsing removed!

You can inspect mobile firefox through remote debugging. You need a Desktop Firefox of the same version as the mobile Firefox you want to debug, a USB cable, and to enable USB debugging for both Android itself and in Firefox for Android. You can destroy your profile data if you don't know what you're doing. You can find documentation at https://firefox-source-docs.mozilla.org/devtools-user/about_colon_debugging/index.html

Getting the debugging connection going can be fiddly; you're better off asking for help in a live forum than in this GitHub issue. try https://chat.mozilla.org/

Once you've got it going

  1. on the mobile browser open about:support
  2. on the desktop about:debugging make sure you've selected the mobile Firefox, not "This Firefox". You should see extensions with names like "Mozilla Android Components - Web Compatibility Interventions"
  3. in the desktop about:debugging page "inspect" the about:support tab on the mobile browser. this opens a new tab with the title "Toolbox (USB)"
  4. paste and run the following code in the toolbox "console" to get a list of Firefox data directories and sizes
function dirlist(dir) {
  let sum = 0;
  for (let f of dir.directoryEntries) {
    if (f.isDirectory()) {
      sum += dirlist(f);
    } else {
      sum += f.fileSize;
//      console.log(f.path, f.fileSize);
    }
  }
  console.log(dir.path+"/", sum);
  return sum;
}

var profile = Services.dirsvc.get("ProfD", Ci.nsIFile);
var mozdir = profile.parent.parent.parent;

dirlist(profile);

Most of the data is in the profile so this just lists that, but you could switch the last line to dirlist(mozdir); if you want everything. This just lists size by directory, but you can uncomment the log() call that prints individual file sizes if you really want to bury yourself in detail.

@dveditz
Copy link

dveditz commented Oct 16, 2022

Note that whenever anyone gives you a chunk of code to paste and run in a browser developer tools console they might be trying to hack you. Only do this when you can understand the code enough to tell whether it's doing what it's supposed to be doing.

@opusforlife2
Copy link

did you go into about:config and set security.remote_settings.crlite_filters.enabled to true?

@dveditz Yes! Not manually, but as a default setting in Mull (which in turn takes its pref changes from Arkenfox). Since now we know this is non-standard behaviour, can I just clean up that folder by deleting stuff older than some time threshold, like a month?

@laurentlbm
Copy link

did you go into about:config and set security.remote_settings.crlite_filters.enabled to true?

@dveditz Yes! Not manually, but as a default setting in Mull (which in turn takes its pref changes from Arkenfox).

@opusforlife2 @dveditz It's the same for me, I'm also using Mull.

Our situation is probably different from other people's then...

@dveditz
Copy link

dveditz commented Oct 17, 2022

Thanks for confirming, @opusforlife2 and @laurentlbm! I filed bug 1795710 for the CRLite issue, and we'll definitely get that fixed before we enable the feature in release.

@KennethHeal is definitely running Firefox, not Mull, based on the screenshots. The original problem must be something else. Hopefully he can get the remote debugging instructions working from my earlier comment.

@opusforlife2
Copy link

@dveditz More info: the figure I've provided above is from a device running Mull for over a year. They're all filter and filter.stash files like you've mentioned in the bug.

But I have another device with Mull installed for just a couple of months. I checked today, and this one doesn't have all these files. Instead, the crlite.stash file was bloated up to 600+ MB. At some point today, that file was reduced down to less than half an MB, a whopping 1200x decrease in size. Providing this info for completeness's sake.

@KennethHeal
Copy link
Author

KennethHeal commented Oct 17, 2022

Thanks @dveditz for the clear instfructions, and providing a step forward.
I managed to run the debug. As it lists websites I have redacted some of the names, though none of the redacted names seem particularly significant.
console-export-2022-10-17_17-42-39.redacted.txt.gz

@dveditz
Copy link

dveditz commented Oct 18, 2022

I think what we really want is a comparison of a second listing after a period of time when you measured bloat so we can look for differences.

Anything with a website in the name is "site data" and should be deleted when you clear "cookies and site data" or use the "Manage Data..." button to delete data from specific sites. Re-reading your first few comments and screenshots I see there may be a terminology difference between Firefox on Android and Desktop. The "Cookies" option on Android most likely also includes Local Storage and IndexedDB files, because they are used in "cookie-like" ways. If you're not clearing Cookies you could be leaving a huge amount of data behind, and the way some sites work they could definitely be mismanaging their storage and having it grow a lot (a web site bug). There is a quota, but I bet it's a lot more space than you'd be happy with.

When paired with clearing History as in your screenshot I'd have to guess "site data" meant "data about the site", such as the permissions you granted or blocked or the site zoom level, and those don't typically take up a lot of space. they're paired because if you want to clear evidence you went to SiteX by clearing History, having a saved permission for that site (autoplay videos or camera access or whatever) reveals that you went there. Similarly, if you clear cookies but don't clear local storage or vice versa, sites could restore whatever state you were trying to reset by clearing one or the other.

I won't have time to look at your archive tonight and I'm off for the day tomorrow traveling. I hope @cpeterso can find someone from the Android team to look into it.

@dveditz
Copy link

dveditz commented Oct 18, 2022

Couldn't help myself and took a quick look. It looks like our confusing terminology has misled you. The bulk of your profile data (172M out of 226M) is in storage/default, which is data stored by websites. It is completely understandable that you might think data stored by sites would be cleared by the checkbox that includes the text "site data", but "storage" is actually cleared along with Cookies.

The biggest culprit in your list was mobile.twitter.com at 20Mb, but there really was no one culprit -- just a lot of sites using up a lot of your disk. Across all the sites most of the space was in Service Worker caches. The service workers of the top 7 sites were taking up close to 150Mb of that 172. 62 sites in total used the Service Worker cache.

If I got a chance to re-word that "Delete Browsing Data" dialog from your first comment I'd try something like

[ ] Browsing History and site permissions
[ ] Cookies and other data saved by sites

Clumsy perhaps (and "permissions" doesn't cover all of the "data about sites" that's included) but at least it's not ambiguous.

If you delete "Cookies" your storage problems should go away.

Mostly. I see 13Mb stored in "minidumps" (crash reports) and I can't find a way to clear those (short of zapping all app data from the OS settings). I've got a bunch myself. It's a drop in the bucket of what you're worried about, but we should fix that. We do seem to clear them up when submitted on Desktop. This might be a known issue

@cpeterso
Copy link

Corresponding Jira issue: https://mozilla-hub.atlassian.net/browse/FNXV2-21262

@KennethHeal
Copy link
Author

@cpeterso is the JIRA internal only or also visible to outside admins/users/devs? Which of the issues above does the JIRA target?

@KennethHeal
Copy link
Author

Yesterday I deleted cookies and site data for just coinbase.com and ghostarchive.org and indeed managed to recover a bit of space.

Tonight I ran the script against mozdir and got the attached output.
console-export-2022-10-19_20-54-12_redacted.txt.gz

The device showed this usage:
20221019_212227

What strikes me here is the device shows a total usage of 454 MB with 192 MB being app and 262 MB data.
The script shows a total of 337 MB.
So that is a 117 MB differnce which I cannot entirely explain.

@cpeterso
Copy link

Since this app data appears to be from Gecko, not the Fenix app frontend, I'll move this bug report to the GeckoView team to debug.

@cpeterso
Copy link

Moved to bugzilla: https://bugzilla.mozilla.org/show_bug.cgi?id=1796267

Change performed by the Move to Bugzilla add-on.

@dveditz
Copy link

dveditz commented Oct 25, 2022

The stale cert-revocation data is now getting cleaned up in Nightly, and the fix will eventually make it to release.

@dveditz
Copy link

dveditz commented Oct 25, 2022

@cpeterso is the JIRA internal only or also visible to outside admins/users/devs? Which of the issues above does the JIRA target?

The JIRA tickets are not public and are used for project management. It's essentially a pointer to this issue, or now I suppose, to the bugzilla issue Chris opened

@opusforlife2
Copy link

@dveditz Meanwhile, is it okay to clean up the files older than, say, a month?

@KennethHeal
Copy link
Author

@dveditz do you have a script I could use to clear out the minidumps directory? thanks

@dveditz
Copy link

dveditz commented Oct 28, 2022

@dveditz Meanwhile, is it okay to clean up the files older than, say, a month?

Yes. It's safe to delete everything older than the most recent set of a base -filter file plus its -filter.stash updates.

@dveditz do you have a script I could use to clear out the minidumps directory? thanks

I'm sorry, I don't. On desktop Firefox the about:crashes page has a "Clear All" button that does it. The Android version of that page does not have a way to clear the old reports. It needs that feature!

@KennethHeal
Copy link
Author

thanks, Should I open a separate github ticket to request a Clear All button in Fenix about:crashes?

@cpeterso
Copy link

cpeterso commented Nov 12, 2022

thanks, Should I open a separate github ticket to request a Clear All button in Fenix about:crashes?

Please! If you could open a new ticket summarizing the information you shared about the old crash files you found on your device and where, that would be helpful.

Fenix should probably automatically clean up old crashes. After we upload the crash file, there is little reason to keep the file on the local device. A user should not have to manually delete the crash files. :(

@opusforlife2
Copy link

should *not have to 🤭

@KennethHeal
Copy link
Author

Data usage is still horrendous and I notice hugely unaccounted for
Android shows 302mb user data and 192mb application binaries, for a total of 493mb.
Screenshot_20221119-123527

Running the script against mozdir gives me this:
console-export-2022-11-19_12-34-36.redacted.txt.gz

I see
/data/user/0/org.mozilla.firefox/ 385550297 = 368mb
/data/user/0/org.mozilla.firefox/databases/ 35280584 = 34mb
/data/user/0/org.mozilla.firefox/lib/ 101799579 = 97mb
/data/user/0/org.mozilla.firefox/files/mozilla/ 223515352 = 213mb
/data/user/0/org.mozilla.firefox/files/mozilla/4pllbj0b.default/minidumps/ 23662792 = 22.6mb

So none of these figures seem to add up.

@KennethHeal
Copy link
Author

KennethHeal commented Nov 30, 2022

@dveditz I currently have latest firefox on desktop (107.0.1) and Android (107.0.1)
Firefox on Android has currently ballooned to 360mb.

Running the script now gives me this:
Uncaught
Exception { name: "NS_ERROR_FAILURE", message: "Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIFile.directoryEntries]", result: 2147500037, filename: "debugger eval code", lineNumber: 8, columnNumber: 0, data: null, stack: "dirlist@debugger eval code:8:16\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\n@debugger eval code:23:8\ngetEvalResult@resource://devtools/server/actors/webconsole/eval-with-debugger.js:19:104\nexports.evalWithDebugger@resource://devtools/server/actors/webconsole/eval-with-debugger.js:7:23\nevaluateJS@resource://devtools/server/actors/webconsole.js:1130:38\nevaluateJSAsync/<@resource://devtools/server/actors/webconsole.js:1021:35\nexports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:4:203\n", location: XPCWrappedNative_NoHelper }

columnNumber: 0
data: null
filename: "debugger eval code"
lineNumber: 8
location: XPCWrappedNative_NoHelper { QueryInterface: QueryInterface(), filename: Getter, name: Getter, … }
message: "Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIFile.directoryEntries]"
name: "NS_ERROR_FAILURE"
result: 2147500037

stack: "dirlist@debugger eval code:8:16\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\ndirlist@debugger eval code:10:11\n@debugger eval code:23:8\ngetEvalResult@resource://devtools/server/actors/webconsole/eval-with-debugger.js:19:104\nexports.evalWithDebugger@resource://devtools/server/actors/webconsole/eval-with-debugger.js:7:23\nevaluateJS@resource://devtools/server/actors/webconsole.js:1130:38\nevaluateJSAsync/<@resource://devtools/server/actors/webconsole.js:1021:35\nexports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:4:203\n"

: ExceptionPrototype { toString: toString(), name: Getter, message: Getter, … }
debugger eval code:12:4

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Labels
performance Possible performance wins qa-triaged Issues triaged by qa S2 Major Functionality/product severely impaired and a satisfactory workaround doesn't exist
Projects
None yet
Development

No branches or pull requests