Fix: more improvements & fixes

.github/workflows/add_new_or_updated_feeds.yml

@@ -88,27 +88,27 @@ jobs:
           PYTHONIOENCODING: "utf8" #ascii
         shell: bash
         run: |
+            echo -e "\n\n"
             echo "BEGIN VIRTUAL ENV && SETUP PIP"
-            echo -e "\n"
             python -m venv env
             source env/bin/activate
             pip install --upgrade pip
             pip install virtualenv --quiet
             pip install gtfs_kit --quiet
             pip install unidecode --quiet
+            echo "DONE VIRTUAL ENV && SETUP PIP"
+            echo -e "\n\n"
+            echo "BEGIN SEND COMMAND TO PYTHON"
 
-            echo "BEGIN"
-
-            sections=$(echo '${{ steps.process-csv.outputs.PYTHON_SCRIPT_ARGS }}' | sed 's/""/"/g' | sed "s/’/'/g" | awk -F'§' '{for (i=1; i<=NF; i++) print $i}')
-            set -x # this should force GitHub to output every echo commands
+            sections=$(echo '${{ steps.process-csv.outputs.PYTHON_SCRIPT_ARGS }}' | sed 's/""/"/g' | sed "s/’/'/g" | sed 's/"comment": "}/"comment": ""}/g' | awk -F'§' '{for (i=1; i<=NF; i++) print $i}')
             for section in "${sections[@]}"; do
-                echo "processing line: ${section}"
+                # echo "processing line: ${section}"
                 output=$(eval "python -c 'from tools.operations import *; ${section}'")
-                echo "$output"
+                # echo "$output"
             done
-            set +x
 
-            echo "DONE"
+            echo "DONE SEND COMMAND TO PYTHON"
+            echo -e "\n\n"
             git status
 
       - name: Commit, push, and create PR

scripts/process_csv_in_github_action.swift

@@ -178,18 +178,26 @@ struct feed {
 
 // MARK: - MAIN
 
-let args : [String] = CommandLine.arguments // this is for using inside the GitHub workflow only.
-// let args : [String] = [ // this is for local testing purposes only.
-//     "scriptname", 
-//     "https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/gviz/tq?tqx=out:csv;outFileName:data&sheet=%5BCLEANED%5D%20For%20import&range=A2:S", 
-//     "11/11/2024", 
-//     "[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", 
-//     "MM/dd/yyyy"
-// ]
+var args : [String] = [""]
+var isInDebugMode : Bool = true
+
+if CommandLine.arguments.count == 5 {
+    // this is for using inside the GitHub workflow only.
+    print("Running inside GitHub Actions.")
+    args = CommandLine.arguments
+} else {
+    // this is for local testing purposes only.
+    print("Running locally.")
+    args = ["scriptname", 
+        "https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/gviz/tq?tqx=out:csv;outFileName:data&sheet=%5BCLEANED%5D%20For%20import&range=A2:S", 
+        "11/11/2024", 
+        "[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", 
+        "MM/dd/yyyy"
+    ]
+    isInDebugMode = true
+}
 // Google Sheet: https://docs.google.com/spreadsheets/d/1Q96KDppKsn2khdrkraZCQ7T_qRSfwj7WsvqXvuMt4Bc/edit?gid=2061813733#gid=2061813733
 
-// Set to false for production use
-let isInDebugMode : Bool = false
 
 if args.count == 5 {
 
@@ -490,7 +498,7 @@ func parseCSV(csvLines: [String], columnSeparator: String, dateFormatRegex: Stri
             apiKeyParameterName         : csvArrayColumn[column.api_key_parameter_name].trimmingCharacters(in: .whitespacesAndNewlines),
             note                        : csvArrayColumn[column.note].trimmingCharacters(in: .whitespacesAndNewlines),
             status                      : csvArrayColumn[column.status].trimmingCharacters(in: .whitespacesAndNewlines),
-            redirects                   : redirectArray(for: csvArrayColumn[column.redirects].trimmingCharacters(in: .whitespacesAndNewlines).trimmingCharacters(in: .escapedDoubleQuote)),
+            redirects                   : redirectArray(for: csvArrayColumn[column.redirects]),
             dataProducerEmail           : csvArrayColumn[column.dataproduceremail].trimmingCharacters(in: .whitespacesAndNewlines)
         )
 
@@ -573,11 +581,10 @@ func redirectArray(for rawData: String) -> String {
 
     let openingPrefix       : String = ", redirects=["
     let closingSuffix       : String = "]"
-    let prefix              : String = "{\"\"id\"\": "
-    let suffix              : String = ", \"\"comment\"\": \"\"}"
+    let prefix             : String = "{\"\"id\"\": "
+    let suffix             : String = ", \"\"comment\"\": \"\" \"\"}"
     let keyValuePairsJoiner : String = ", "
 
-    // Transform each `currentString` in `rawDataAsArray` with `map` and join them in one step
     let redirectEntries : String = rawData
         .components(separatedBy: defaults.comma)
         .map { prefix + $0 + suffix }
@@ -668,10 +675,10 @@ func prettyPrintPythonCommands(input: String) -> String {
             .joined(separator: defaults.comma + defaults.newline)
 
         // Rebuild the command with the formatted arguments
-        let formattedCommand = "\(functionName)(\n\(formattedArguments)\n)"
+        let formattedCommand : String = "\(functionName)(\n\(formattedArguments)\n)"
 
         // Fix the specific issue with comment formatting
-        return formattedCommand.replacingOccurrences(of: ",\n\t\"\"comment\": \"\"}]", with: ", \"\"comment\": \"\"}]")
+        return formattedCommand //.replacingOccurrences(of: ",\n\t\"\"comment\": \"\"}]", with: ", \"\"comment\": \"\"}]")
     }
 
     // Join all the formatted commands with two newlines