util: timeline network conversion script [wip]

[quisquous]

I'm not intending to check this in, but this is as a reference.

You can run this via ts-node util/nettify.ts ui/raidboss/data/, although you should apply #5975 first

This is enough to at least satisfy npm run test (checking key names, and net log def names and general json formatting), but it almost certainly needs manual testing and a closer eye.

I think a follow-up could be to split name: "(a|b)" into name: ["a", "b"] as well, but I didn't want to do that here because it would be impossible to find in the noise.

My vague plan (please disagree) is to do some more manual testing, and then do one commit for each of the replacement types (e.g. ability, startsusing, etc), and at some point combine them when the numbers get smaller.

[quisquous]

Here's the script output with ~3 examples of each kind of replacement it did, to verify that things look ok.

ability: 29449

-8.0 "Crackle Hiss" sync / 1[56]:[^:]*:Imdugud:B55:/
+8.0 "Crackle Hiss" Ability { id: "B55", source: "Imdugud" }
-16.0 "Critical Rip" sync / 1[56]:[^:]*:Imdugud:B56:/
+16.0 "Critical Rip" Ability { id: "B56", source: "Imdugud" }
-20.0 "Crackle Hiss" sync / 1[56]:[^:]*:Imdugud:B55:/
+20.0 "Crackle Hiss" Ability { id: "B55", source: "Imdugud" }

startsUsing: 816

-196.0 "--sync--" sync / 14:[^:]*:Imdugud:B5D:/ window 200,0
+196.0 "--sync--" StartsUsing { id: "B5D", source: "Imdugud" } window 200,0
-497.0 "--sync--" sync / 14:[^:]*:Imdugud:B5E:/ window 500,0
+497.0 "--sync--" StartsUsing { id: "B5E", source: "Imdugud" } window 500,0
-696.0 "--sync--" sync / 14:[^:]*:Imdugud:B5D:/ window 200,0
+696.0 "--sync--" StartsUsing { id: "B5D", source: "Imdugud" } window 200,0

systemLog: 297

-0.0 "--sync--" sync / 29:[^:]*:7DC:[^:]*:588:/ window 5,5
+0.0 "--sync--" SystemLogMessage { id: "7DC", param1: "588" } window 5,5
-0.0 "--sync--" sync / 29:[^:]*:7DC:[^:]*:592:/ window 10,10
+0.0 "--sync--" SystemLogMessage { id: "7DC", param1: "592" } window 10,10
-0.0 "--sync--" sync / 29:[^:]*:7DC:[^:]*:51C:/ window 10,10
+0.0 "--sync--" SystemLogMessage { id: "7DC", param1: "51C" } window 10,10

backslashEscaper: 222

-45.8 "E.D.D. Add" AddedCombatant { name: "E\.D\.D\." } window 50,5
+45.8 "E.D.D. Add" AddedCombatant { name: "E\\.D\\.D\\." } window 50,5
-458.7 "E.D.D. Add" AddedCombatant { name: "E\.D\.D\." }
+458.7 "E.D.D. Add" AddedCombatant { name: "E\\.D\\.D\\." }
-697.2 "E.D.D. Add" AddedCombatant { name: "E\.D\.D\." }
+697.2 "E.D.D. Add" AddedCombatant { name: "E\\.D\\.D\\." }

addedCombatant: 149

-0.0 "6x Bug" sync / 03:........:Clockwork Bug:/  window 0,1
+0.0 "6x Bug" AddedCombatant { name: "Clockwork Bug" }  window 0,1
-63.0 "Knight + Soldier (NW)" sync / 03:........:Clockwork Knight:/  window 70,0
+63.0 "Knight + Soldier (NW)" AddedCombatant { name: "Clockwork Knight" }  window 70,0
-123.0 "Dreadnaught (NW)" sync / 03:........:Clockwork Dreadnaught:/  window 130,0
+123.0 "Dreadnaught (NW)" AddedCombatant { name: "Clockwork Dreadnaught" }  window 130,0

inCombat: 146

-0.0 "--sync--" sync / 104:[^:]*:1($|:)/ window 0,1
+0.0 "--sync--" InCombat { inGameCombat: "1" } window 0,1
-0.0 "--sync--" sync / 104:[^:]*:1($|:)/ window 0,1
+0.0 "--sync--" InCombat { inGameCombat: "1" } window 0,1
-0.0 "--sync--" sync / 104:[^:]*:1($|:)/ window 0,1
+0.0 "--sync--" InCombat { inGameCombat: "1" } window 0,1

nameToggle: 49

-25.8 "--untargetable--" sync / 22:........:Leviathan:........:Leviathan:00/ window 30,10
+25.8 "--untargetable--" NameToggle { name: "Leviathan", toggle: "00" } window 30,10
-3300.0 "--untargetable--" sync / 22:........:Scathach:........:Scathach:00/ window 300,5
+3300.0 "--untargetable--" NameToggle { name: "Scathach", toggle: "00" } window 300,5
-4274.0 "--sync--" sync / 22:........:Diabolos:........:Diabolos:00/ window 100,5
+4274.0 "--sync--" NameToggle { name: "Diabolos", toggle: "00" } window 100,5

actorControlNoData: 48

-0.0 "--Reset--" sync / 21:........:4000000F:/ window 100000 jump 0
+0.0 "--Reset--" ActorControl { command: "4000000F" } window 100000 jump 0
-0.0 "--Reset--" sync / 21:........:4000000F:/ window 100000 jump 0
+0.0 "--Reset--" ActorControl { command: "4000000F" } window 100000 jump 0
-0.0 "--Reset--" sync / 21:........:4000000F:/ window 100000 jump 0
+0.0 "--Reset--" ActorControl { command: "4000000F" } window 100000 jump 0

actorControl: 34

-0.0 "--Reset--" sync / 21:........:80000014:00:/ window 100000 jump 0
+0.0 "--Reset--" ActorControl { command: "80000014", data0: "00" } window 100000 jump 0
-1000.0 "--sync--" sync / 21:........:80000014:1D4:/ window 100000,0
+1000.0 "--sync--" ActorControl { command: "80000014", data0: "1D4" } window 100000,0
-2000.0 "--sync--" sync / 21:........:80000014:1CC:/ window 100000,0
+2000.0 "--sync--" ActorControl { command: "80000014", data0: "1CC" } window 100000,0

systemLogNoParam: 32

-0.0 "--Reset--" sync / 29:[^:]*:7DE:/ window 100000 jump 0
+0.0 "--Reset--" SystemLogMessage { id: "7DE" } window 100000 jump 0
-0.0 "--Reset--" sync / 29:[^:]*:7DE:/ window 100000 jump 0
+0.0 "--Reset--" SystemLogMessage { id: "7DE" } window 100000 jump 0
-0.0 "--Reset--" sync / 29:[^:]*:7DE:/ window 100000 jump 0
+0.0 "--Reset--" SystemLogMessage { id: "7DE" } window 100000 jump 0

gameLogWithName: 24

-200.0 "--sync--" sync / 00:0044:Ifrit:Succumb to the inferno/ window 200,0
+200.0 "--sync--" GameLog { code: "0044", name: "Ifrit", line: "Succumb to the inferno.*?" } window 200,0
-300.0 "--sync--" sync / 00:0044:Ifrit:Surrender thyself to the fires of judgment/ window 300,0
+300.0 "--sync--" GameLog { code: "0044", name: "Ifrit", line: "Surrender thyself to the fires of judgment.*?" } window 300,0
-1053.4 "--sync--" sync / 00:0044:Vocal Guidance System:This humanoid prototype can perfectly replicate/ window 53.4,10 jump 1100.0
+1053.4 "--sync--" GameLog { code: "0044", name: "Vocal Guidance System", line: "This humanoid prototype can perfectly replicate.*?" } window 53.4,10 jump 1100.0

abilityNoSource: 22

-451.8 "--sync--" sync / 1[56]:[^:]*:[^:]*:1059:/ # All the knights use this unknown attack here
+451.8 "--sync--" Ability { id: "1059" } # All the knights use this unknown attack here
-800.0 "Freefire" sync / 1[56]:[^:]*:[^:]*:2CF5:/ window 800,0
+800.0 "Freefire" Ability { id: "2CF5" } window 800,0
-1625.0 "Citadel Siege 1" sync / 1[56]:[^:]*:[^:]*:2B92:/ window 30,0
+1625.0 "Citadel Siege 1" Ability { id: "2B92" } window 30,0

gameLogNoName: 12

-0 "--Reset--" sync / 00:001D:[^:]*:You bid farewell to the striking dummy/ window 10000 jump 0
+0 "--Reset--" GameLog { code: "001D", line: "You bid farewell to the striking dummy.*?" } window 10000 jump 0
-0 "--sync--" sync / 00:0039:[^:]*:Engage!/ window 100000,100000
+0 "--sync--" GameLog { code: "0039", line: "Engage!.*?" } window 100000,100000
-0 "--sync--" sync / 00:001D:[^:]*:You bow courteously to the striking dummy/ window 0,1
+0 "--sync--" GameLog { code: "001D", line: "You bow courteously to the striking dummy.*?" } window 0,1

gainsEffectWithTarget: 9

-200 "--sync--" sync / 1A:5D1:Ultros Simulation:[^:]*:[^:]*:[^:]*:[^:]*:Guardian:/ window 2000,2000
+200 "--sync--" GainsEffect { effectId: "5D1", effect: "Ultros Simulation", target: "Guardian" } window 2000,2000
-400 "--sync--" sync / 1A:5D3:Dadaluma Simulation:[^:]*:[^:]*:[^:]*:[^:]*:Guardian:/ window 2000,2000
+400 "--sync--" GainsEffect { effectId: "5D3", effect: "Dadaluma Simulation", target: "Guardian" } window 2000,2000
-600 "--sync--" sync / 1A:5D2:Air Force Simulation:[^:]*:[^:]*:[^:]*:[^:]*:Guardian:/ window 2000,2000
+600 "--sync--" GainsEffect { effectId: "5D2", effect: "Air Force Simulation", target: "Guardian" } window 2000,2000

mapEffectNoInstanceNoLocation: 6

-12.9 "Wildlife Crossing 1" sync / 257 101:........:00020001:/ window 5,5 duration 15
+12.9 "Wildlife Crossing 1" MapEffect { flags: "00020001" } window 5,5 duration 15
-26.9 "Wildlife Crossing 2" sync / 257 101:........:00020001:/ window 5,5 duration 15
+26.9 "Wildlife Crossing 2" MapEffect { flags: "00020001" } window 5,5 duration 15
-106.1 "Wildlife Crossing 1" sync / 257 101:........:00020001:/ window 5,5 duration 15
+106.1 "Wildlife Crossing 1" MapEffect { flags: "00020001" } window 5,5 duration 15

gainsEffectNameOnly: 4

-84 "Resync" sync / 1A:[^:]*:Pyretic:/ window 20 jump 384
+84 "Resync" GainsEffect { effect: "Pyretic" } window 20 jump 384
-582 "Resync" sync / 1A:[^:]*:Frostbite:/ window 20 jump 983
+582 "Resync" GainsEffect { effect: "Frostbite" } window 20 jump 983
-1348 "Resync" sync / 1A:[^:]*:Pyretic:/ window 20 jump 1548
+1348 "Resync" GainsEffect { effect: "Pyretic" } window 20 jump 1548

mapEffect: 4

-3062.1 "--sync--" sync / 257 101:80038CA1:00020001:09:/ window 70,70 jump 3262.1
+3062.1 "--sync--" MapEffect { instance: "80038CA1", flags: "00020001", location: "09" } window 70,70 jump 3262.1
-3062.1 "--sync--" sync / 257 101:80038CA1:20000004:0A:/ window 70,70 jump 4062.1
+3062.1 "--sync--" MapEffect { instance: "80038CA1", flags: "20000004", location: "0A" } window 70,70 jump 4062.1
-3262.1 "--sync--" sync / 257 101:80038CA1:00020001:09:/
+3262.1 "--sync--" MapEffect { instance: "80038CA1", flags: "00020001", location: "09" }

headMarkerNoName: 3

-503.3 "Golem Meteors" sync / 1B:........:[^:]*:....:....:0007:/ duration 11 window 505,0
+503.3 "Golem Meteors" HeadMarker { id: "0007" } duration 11 window 505,0
-582.3 "Meteor 1/6" duration 7 sync / 1B:........:[^:]*:....:....:000[9A]:/ window 83,0
+582.3 "Meteor 1/6" duration 7 HeadMarker { id: "000[9A]" } window 83,0
-598.3 "Golem Meteors" sync / 1B:........:[^:]*:....:....:0007:/ duration 11 window 80,80
+598.3 "Golem Meteors" HeadMarker { id: "0007" } duration 11 window 80,80

gameLogNoNameNoCode: 3

-1100.0 "--sync--" sync / 00:[^:]*::The liquid flame gains the effect of Chiromorph/ window 100,250
+1100.0 "--sync--" GameLog { line: "The liquid flame gains the effect of Chiromorph.*?" } window 100,250
-1200.0 "--sync--" sync / 00:[^:]*::The liquid flame gains the effect of Anemomorph/ window 200,0
+1200.0 "--sync--" GameLog { line: "The liquid flame gains the effect of Anemomorph.*?" } window 200,0
-1300.0 "--sync--" sync / 00:[^:]*::The liquid flame gains the effect of Anthropomorph/ window 300,10
+1300.0 "--sync--" GameLog { line: "The liquid flame gains the effect of Anthropomorph.*?" } window 300,10

interrupt: 3

-3154.9 "--sync--" sync / 17:[^:]*:Raiden:3878:/ window 40,0
+3154.9 "--sync--" NetworkCancelAbility { id: "3878", source: "Raiden" } window 40,0
-3341.7 "--sync--" sync / 17:[^:]*:Raiden:3878:/ window 40,0
+3341.7 "--sync--" NetworkCancelAbility { id: "3878", source: "Raiden" } window 40,0
-993.1 "--sync--" sync / 17:[^:]*:King Thordan:63C3:/ window 300,0
+993.1 "--sync--" NetworkCancelAbility { id: "63C3", source: "King Thordan" } window 300,0

headMarkerName: 2

-1048.0  "--sync--" sync / 1B:........:Minotaur:....:....:0036:/ window 45,15 jump 1151.0
+1048.0  "--sync--" HeadMarker { id: "0036", target: "Minotaur" } window 45,15 jump 1151.0
-1219.0 "--sync--" sync / 1B:........:Minotaur:....:....:0036:/ window 45,15 jump 1151.0
+1219.0 "--sync--" HeadMarker { id: "0036", target: "Minotaur" } window 45,15 jump 1151.0

removedCombatant: 2

-0.0 "--Reset--" sync / 04:........:Proto Ozma:/ window 100000 jump 0
+0.0 "--Reset--" RemovedCombatant { name: "Proto Ozma" } window 100000 jump 0
-196.0 "--sync--" sync / 04:........:Liquid Hand:/ window 50,0
+196.0 "--sync--" RemovedCombatant { name: "Liquid Hand" } window 50,0

abilityWithNonEmptyTarget: 2

-1002.5 "--sync--" sync / 1[56]:[^:]*:Art:3956:[^:]*:[^:]*:[^:]+:/ window 1500,0
+1002.5 "--sync--" Ability { id: "3956", source: "Art", target: "[:^]+" } window 1500,0
-2002.5 "--sync--" sync / 1[56]:[^:]*:Owain:3957:[^:]*:[^:]*:[^:]+:/ window 2500,0
+2002.5 "--sync--" Ability { id: "3957", source: "Owain", target: "[:^]+" } window 2500,0

wasDefeated: 1

-6000.0 "--Reset--" sync / 19:[^:]*:Dahu:/ window 0,2000 jump 0
+6000.0 "--Reset--" WasDefeated { target: "Dahu" } window 0,2000 jump 0

addedCombatantOneOff: 1

-1.0 "--sync--" sync / 03:[^:]*:Hemitheos:00:5A:0{4}:00::12383:/ window 10,3 jump 1000.5 # Sync to P2 immediately through AddCombatant.
+1.0 "--sync--" AddedCombatant { npcNameId: "12383", name: "Hemitheos", job: "00", level: "5A", ownerId: "0{4}", worldId: "00" } window 10,3 jump 1000.5 # Sync to P2 immediately through AddCombatant.

[JLGarber]

I'm not going to be able to do a full review tonight, that's just too many files! I did look at a subset that seemed likely to have issues and I didn't notice any issues from the script run. (Fractal normal/hard, Bardam's Mettle, Gubal hard, A7s, A8s, Shinryu, and O12n. All of these have peculiarities with their timelines that make them likely candidates for nonsense.)

I think your plan here is solid. If test doesn't seem to be reporting any issues, I think this script is very close to being what we want.

[valarnin]

Isn't the normal process for mass changes like this to do them by expansion instead? So it's easier to do a manual review and test?

[quisquous]

Isn't the normal process for mass changes like this to do them by expansion instead? So it's easier to do a manual review and test?

I feel like there's multiple dimensions here of both "a lot to review" and also "all the lines changing". I thought if I change all the lines at once, it's basically impossible to review because you have to look at everything, vs committing a change where all the lines of a certain type change at once. This is also more automatic (entirely automatic?) than a lot of other large scale changes.

I might do ability and startsusing by expansion, and then do one line type at a time after that. I feel pretty confident on the ability/startsusing lines because they're pretty narrow regexes, but I worry that maybe there's some edge cases lurking in the other lines that I didn't look closely enough at (or I messed up the regex) and having them be separate changes will make it much easier to look through the <100 examples of them.

I don't know, that's my thought, at least.

[valarnin]

I feel like there's multiple dimensions here of both "a lot to review" and also "all the lines changing". I thought if I change all the lines at once, it's basically impossible to review because you have to look at everything, vs committing a change where all the lines of a certain type change at once. This is also more automatic (entirely automatic?) than a lot of other large scale changes.

I think this is maybe just a difference in opinion? I find it mentally easier to deal with looking at the timeline when all the syncs are in the same format.

I might do ability and startsusing by expansion, and then do one line type at a time after that. I feel pretty confident on the ability/startsusing lines because they're pretty narrow regexes, but I worry that maybe there's some edge cases lurking in the other lines that I didn't look closely enough at (or I messed up the regex) and having them be separate changes will make it much easier to look through the <100 examples of them.

I think this should work pretty well? My main concern here was from a PR size perspective, changing every ability line across every expansion would be 26001 lines of change in a single PR.

Even breaking it down by expansion and only for Ability/StartsUsing lines in timelines it's still pretty big:

• 02-arr: 1366 lines
• 03-hw: 3241 lines
• 04-sb: 5824 lines
• 05-shb: 6555 lines
• 06-ew: 9015 lines

(numbers checked with grep -rP '^[^#]+sync[^#]+ 1(?:[456]|\[[456]+\])' --include \*.txt "02-arr" | wc -l which should be pretty thorough)

It may also make sense to flag potential "troublesome" timeline entries for a more deep look, e.g. any field that doesn't match ^[a-zA-Z0-9]+$. Concerns here include characters that may need escaped (', ", \, /, etc), or unicode characters such as æ in Pandæmonium.

[quisquous]

It may also make sense to flag potential "troublesome" timeline entries for a more deep look, e.g. any field that doesn't match ^[a-zA-Z0-9]+$. Concerns here include characters that may need escaped (', ", , /, etc), or unicode characters such as æ in Pandæmonium.

Re: these characters, I will call out anything that gets backslashes in the PR. So far this is shb/ew for ability. Pandæmonium also has a backslash, so is listed there. I feel fairly confident about this, largely due to the translation issues (which is why I added the backslash escaper in the first place). If the translation replacements for these names continue working, then it's likely that the names are still the same as they were (I think?)

The remaining lines are shb/ew startsusing, and one hw addedcombatant case:

-45.8 "E.D.D. Add" AddedCombatant { name: "E\.D\.D\." } window 50,5
+45.8 "E.D.D. Add" AddedCombatant { name: "E\\.D\\.D\\." } window 50,5
-458.7 "E.D.D. Add" AddedCombatant { name: "E\.D\.D\." }
+458.7 "E.D.D. Add" AddedCombatant { name: "E\\.D\\.D\\." }
-697.2 "E.D.D. Add" AddedCombatant { name: "E\.D\.D\." }
+697.2 "E.D.D. Add" AddedCombatant { name: "E\\.D\\.D\\." }
-3000.0 "--sync--" StartsUsing { id: "E49", source: "Oppressor 0\.5" } window 3000,0
+3000.0 "--sync--" StartsUsing { id: "E49", source: "Oppressor 0\\.5" } window 3000,0
-2172.7 "--sync--" StartsUsing { id: "4FC4", source: "724P-Operated Superior Flight Unit \(A-Lpha\)" } window 70,20
+2172.7 "--sync--" StartsUsing { id: "4FC4", source: "724P-Operated Superior Flight Unit \\(A-Lpha\\)" } window 70,20
-2172.7 "--sync--" StartsUsing { id: "4FC4", source: "767P-Operated Superior Flight Unit \(B-Eta\)" } window 70,20
+2172.7 "--sync--" StartsUsing { id: "4FC4", source: "767P-Operated Superior Flight Unit \\(B-Eta\\)" } window 70,20
-2172.7 "--sync--" StartsUsing { id: "4FC4", source: "772P-Operated Superior Flight Unit \(C-Hi\)" } window 70,20
+2172.7 "--sync--" StartsUsing { id: "4FC4", source: "772P-Operated Superior Flight Unit \\(C-Hi\\)" } window 70,20
-2427.2 "--sync--" StartsUsing { id: "4FBD", source: "724P-Operated Superior Flight Unit \(A-Lpha\)" } window 70,20
+2427.2 "--sync--" StartsUsing { id: "4FBD", source: "724P-Operated Superior Flight Unit \\(A-Lpha\\)" } window 70,20
-2427.2 "--sync--" StartsUsing { id: "4FBD", source: "767P-Operated Superior Flight Unit \(B-Eta\)" } window 70,20
+2427.2 "--sync--" StartsUsing { id: "4FBD", source: "767P-Operated Superior Flight Unit \\(B-Eta\\)" } window 70,20
-2427.2 "--sync--" StartsUsing { id: "4FBD", source: "772P-Operated Superior Flight Unit \(C-Hi\)" } window 70,20
+2427.2 "--sync--" StartsUsing { id: "4FBD", source: "772P-Operated Superior Flight Unit \\(C-Hi\\)" } window 70,20
-7.9 "--sync--" StartsUsing { id: "8259", source: "Pand\u00e6monium" } window 15,15
+7.9 "--sync--" StartsUsing { id: "8259", source: "Pand\\u00e6monium" } window 15,15
-9.1 "--sync--" StartsUsing { id: "82A5", source: "Pand\u00e6monium" } window 15,15
+9.1 "--sync--" StartsUsing { id: "82A5", source: "Pand\\u00e6monium" } window 15,15

[quisquous]

I caught ~3 issues with manual review here. If anybody wants to double check any work on this conversion, I think the two that could just use a double check (or a log if you have it) are these 3 lines from 2 categories:

• raidboss: timeline netregex for abilityWithNonEmptyTarget #6036
• raidboss: timeline netregex for addedCombatantOneOff #6038

[quisquous]

Also for reference:

  sourceGroups: (line) => {
    // 213.0 "--sync--" Ability { id: "7E9", source: "(Astral Debris|Umbral Debris)" }
    const regex = /(?<=\w* {[^}]*source: )"\((?<source>[^)]*)\)"/;
    const source = regex.exec(line)?.groups?.source;
    if (source === undefined)
      return;
    const sources = source.split('|').map((x) => `"${x.replace('"', '\\"')}"`);
    return line.replace(regex, `[${sources.join(', ')}]`);
  },
  idGroups: (line) => {
    // 873.7 "Superliminal Steel/Optimized Blizzard III" Ability { id: "(7B2A|7B2D)", source: "Omega-M" }
    const regex = /(?<=\w* {[^}]*id: )"\((?<id>[^)]*)\)"/;
    const id = regex.exec(line)?.groups?.id;
    if (id === undefined)
      return;
    const ids = id.split('|').map((x) => `"${x.replace('"', '\\"')}"`);
    return line.replace(regex, `[${ids.join(', ')}]`);
  },