diff --git a/assets/js/phoenix_live_view/constants.js b/assets/js/phoenix_live_view/constants.js
index 2c687e1008..e0feb4575a 100644
--- a/assets/js/phoenix_live_view/constants.js
+++ b/assets/js/phoenix_live_view/constants.js
@@ -60,6 +60,7 @@ export const PHX_AUTO_RECOVER = "auto-recover"
 export const PHX_LV_DEBUG = "phx:live-socket:debug"
 export const PHX_LV_PROFILE = "phx:live-socket:profiling"
 export const PHX_LV_LATENCY_SIM = "phx:live-socket:latency-sim"
+export const PHX_LV_HISTORY_POSITION = "phx:nav-history-position"
 export const PHX_PROGRESS = "progress"
 export const PHX_MOUNTED = "mounted"
 export const PHX_RELOAD_STATUS = "__phoenix_reload_status__"
diff --git a/assets/js/phoenix_live_view/live_socket.js b/assets/js/phoenix_live_view/live_socket.js
index 0d7cdfe9df..1052d60b14 100644
--- a/assets/js/phoenix_live_view/live_socket.js
+++ b/assets/js/phoenix_live_view/live_socket.js
@@ -88,6 +88,7 @@ import {
   PHX_LV_DEBUG,
   PHX_LV_LATENCY_SIM,
   PHX_LV_PROFILE,
+  PHX_LV_HISTORY_POSITION,
   PHX_MAIN,
   PHX_PARENT_ID,
   PHX_VIEW_SELECTOR,
@@ -169,6 +170,7 @@ export default class LiveSocket {
       onBeforeElUpdated: closure()},
     opts.dom || {})
     this.transitions = new TransitionSet()
+    this.currentHistoryPosition = parseInt(this.sessionStorage.getItem(PHX_LV_HISTORY_POSITION)) || 0
     window.addEventListener("pagehide", _e => {
       this.unloaded = true
     })
@@ -687,10 +689,19 @@ export default class LiveSocket {
     })
     window.addEventListener("popstate", event => {
       if(!this.registerNewLocation(window.location)){ return }
-      let {type, id, root, scroll} = event.state || {}
+      let {type, backType, id, root, scroll, position} = event.state || {}
       let href = window.location.href
 
-      DOM.dispatchEvent(window, "phx:navigate", {detail: {href, patch: type === "patch", pop: true}})
+      // Compare positions to determine direction
+      let isForward = position > this.currentHistoryPosition
+
+      type = isForward ? type : (backType || type)
+
+      // Update current position
+      this.currentHistoryPosition = position || 0
+      this.sessionStorage.setItem(PHX_LV_HISTORY_POSITION, this.currentHistoryPosition.toString())
+
+      DOM.dispatchEvent(window, "phx:navigate", {detail: {href, patch: type === "patch", pop: true, direction: isForward ? "forward" : "backward"}})
       this.requestDOMUpdate(() => {
         if(this.main.isConnected() && (type === "patch" && id === this.main.id)){
           this.main.pushLinkPatch(event, href, null, () => {
@@ -769,8 +780,20 @@ export default class LiveSocket {
   historyPatch(href, linkState, linkRef = this.setPendingLink(href)){
     if(!this.commitPendingLink(linkRef)){ return }
 
-    Browser.pushState(linkState, {type: "patch", id: this.main.id}, href)
-    DOM.dispatchEvent(window, "phx:navigate", {detail: {patch: true, href, pop: false}})
+    // Increment position for new state
+    this.currentHistoryPosition++
+    this.sessionStorage.setItem(PHX_LV_HISTORY_POSITION, this.currentHistoryPosition.toString())
+
+    // store the type for back navigation
+    Browser.updateCurrentState((state) => ({...state, backType: "patch"}))
+
+    Browser.pushState(linkState, {
+      type: "patch",
+      id: this.main.id,
+      position: this.currentHistoryPosition
+    }, href)
+
+    DOM.dispatchEvent(window, "phx:navigate", {detail: {patch: true, href, pop: false, direction: "forward"}})
     this.registerNewLocation(window.location)
   }
 
@@ -787,8 +810,21 @@ export default class LiveSocket {
     this.withPageLoading({to: href, kind: "redirect"}, done => {
       this.replaceMain(href, flash, (linkRef) => {
         if(linkRef === this.linkRef){
-          Browser.pushState(linkState, {type: "redirect", id: this.main.id, scroll: scroll}, href)
-          DOM.dispatchEvent(window, "phx:navigate", {detail: {href, patch: false, pop: false}})
+          // Increment position for new state
+          this.currentHistoryPosition++
+          this.sessionStorage.setItem(PHX_LV_HISTORY_POSITION, this.currentHistoryPosition.toString())
+
+          // store the type for back navigation
+          Browser.updateCurrentState((state) => ({...state, backType: "redirect"}))
+
+          Browser.pushState(linkState, {
+            type: "redirect",
+            id: this.main.id,
+            scroll: scroll,
+            position: this.currentHistoryPosition
+          }, href)
+
+          DOM.dispatchEvent(window, "phx:navigate", {detail: {href, patch: false, pop: false, direction: "forward"}})
           this.registerNewLocation(window.location)
         }
         done()
@@ -797,7 +833,12 @@ export default class LiveSocket {
   }
 
   replaceRootHistory(){
-    Browser.pushState("replace", {root: true, type: "patch", id: this.main.id})
+    Browser.pushState("replace", {
+      root: true,
+      type: "patch",
+      id: this.main.id,
+      position: this.currentHistoryPosition // Preserve current position
+    })
   }
 
   registerNewLocation(newLocation){
diff --git a/assets/js/phoenix_live_view/view.js b/assets/js/phoenix_live_view/view.js
index ee7d409ee6..7670e3c42d 100644
--- a/assets/js/phoenix_live_view/view.js
+++ b/assets/js/phoenix_live_view/view.js
@@ -316,6 +316,10 @@ export default class View {
     if(this.root === this){
       this.formsForRecovery = this.getFormsForRecovery()
     }
+    if(this.isMain() && window.history.state === null){
+      // set initial history entry if this is the first page load
+      this.liveSocket.replaceRootHistory()
+    }
 
     if(liveview_version !== this.liveSocket.version()){
       console.error(`LiveView asset version mismatch. JavaScript version ${this.liveSocket.version()} vs. server ${liveview_version}. To avoid issues, please ensure that your assets use the same version as the server.`)
diff --git a/assets/test/live_socket_test.js b/assets/test/live_socket_test.js
index 909175f16c..9bf96177f1 100644
--- a/assets/test/live_socket_test.js
+++ b/assets/test/live_socket_test.js
@@ -224,6 +224,7 @@ describe("LiveSocket", () => {
     let liveSocket = new LiveSocket("/live", Socket, {sessionStorage: override})
     liveSocket.getLatencySim()
 
-    expect(getItemCalls).toEqual(1)
+    // liveSocket constructor reads nav history position from sessionStorage
+    expect(getItemCalls).toEqual(2)
   })
 })
diff --git a/test/e2e/support/navigation.ex b/test/e2e/support/navigation.ex
index fc6ceaf32c..fb7cc7ce9f 100644
--- a/test/e2e/support/navigation.ex
+++ b/test/e2e/support/navigation.ex
@@ -12,6 +12,10 @@ defmodule Phoenix.LiveViewTest.E2E.Navigation.Layout do
       let liveSocket = new LiveSocket("/live", window.Phoenix.Socket, {params: {_csrf_token: csrfToken}})
       liveSocket.connect()
       window.liveSocket = liveSocket
+
+      window.addEventListener("phx:navigate", (e) => {
+        console.log("navigate event", JSON.stringify(e.detail))
+      })
     </script>
 
     <style>
diff --git a/test/e2e/tests/navigation.spec.js b/test/e2e/tests/navigation.spec.js
index a3445507b2..b3b735915d 100644
--- a/test/e2e/tests/navigation.spec.js
+++ b/test/e2e/tests/navigation.spec.js
@@ -225,3 +225,92 @@ test("scrolls hash el into view after live navigation (issue #3452)", async ({pa
   expect(scrollTop).toBeGreaterThanOrEqual(offset - 500)
   expect(scrollTop).toBeLessThanOrEqual(offset + 500)
 })
+
+test("navigating all the way back works without remounting (only patching)", async ({page}) => {
+  await page.goto("/navigation/a")
+  await syncLV(page)
+  networkEvents = []
+  await page.getByRole("link", {name: "Patch this LiveView"}).click()
+  await syncLV(page)
+  await page.goBack()
+  await syncLV(page)
+  expect(networkEvents).toEqual([])
+  // we only expect patch navigation
+  expect(webSocketEvents.filter(e => e.payload.indexOf("phx_leave") !== -1)).toHaveLength(0)
+  // we patched 2 times
+  expect(webSocketEvents.filter(e => e.payload.indexOf("live_patch") !== -1)).toHaveLength(2)
+})
+
+// see https://github.com/phoenixframework/phoenix_live_view/pull/3513
+// see https://github.com/phoenixframework/phoenix_live_view/issues/3536
+test("back and forward navigation types are tracked", async ({page}) => {
+  let consoleMessages = []
+  page.on("console", msg => consoleMessages.push(msg.text()))
+  const getNavigationEvent = () => {
+    const ev = consoleMessages.find((e) => e.startsWith("navigate event"))
+    consoleMessages = []
+    return JSON.parse(ev.slice(15))
+  }
+  // initial page visit
+  await page.goto("/navigation/b")
+  await syncLV(page)
+  consoleMessages = []
+  networkEvents = []
+  // type: redirect
+  await page.getByRole("link", {name: "LiveView A"}).click()
+  await syncLV(page)
+  expect(getNavigationEvent()).toEqual({
+    href: "http://localhost:4004/navigation/a",
+    patch: false,
+    pop: false,
+    direction: "forward"
+  })
+  // type: patch
+  await page.getByRole("link", {name: "Patch this LiveView"}).click()
+  await syncLV(page)
+  expect(getNavigationEvent()).toEqual({
+    href: expect.stringMatching(/\/navigation\/a\?param=.*/),
+    patch: true,
+    pop: false,
+    direction: "forward"
+  })
+  // back should also be type: patch
+  await page.goBack()
+  await expect(page).toHaveURL("/navigation/a")
+  expect(getNavigationEvent()).toEqual({
+    href: "http://localhost:4004/navigation/a",
+    patch: true,
+    pop: true,
+    direction: "backward"
+  })
+  await page.goBack()
+  await expect(page).toHaveURL("/navigation/b")
+  expect(getNavigationEvent()).toEqual({
+    href: "http://localhost:4004/navigation/b",
+    patch: false,
+    pop: true,
+    direction: "backward"
+  })
+  await page.goForward()
+  await expect(page).toHaveURL("/navigation/a")
+  expect(getNavigationEvent()).toEqual({
+    href: "http://localhost:4004/navigation/a",
+    patch: false,
+    pop: true,
+    direction: "forward"
+  })
+  await page.goForward()
+  await expect(page).toHaveURL(/\/navigation\/a\?param=.*/)
+  expect(getNavigationEvent()).toEqual({
+    href: expect.stringMatching(/\/navigation\/a\?param=.*/),
+    patch: true,
+    pop: true,
+    direction: "forward"
+  })
+  // we don't expect any full page reloads
+  expect(networkEvents).toEqual([])
+  // we only expect 3 navigate navigations (from b to a, back from a to b, back to a)
+  expect(webSocketEvents.filter(e => e.payload.indexOf("phx_leave") !== -1)).toHaveLength(3)
+  // we expect 3 patches (a to a with param, back to a, back to a with param)
+  expect(webSocketEvents.filter(e => e.payload.indexOf("live_patch") !== -1)).toHaveLength(3)
+})