Smartsheet reconciliation & attachments

.cfignore

@@ -4,6 +4,7 @@ deployment_config/
 frontend/
 src/
 maintenance_page/
+smartsheet_scripts/
 terraform/
 hses.zip
 temp/

.circleci/config.yml

@@ -164,7 +164,7 @@ parameters:
     default: "main"
     type: string
   sandbox_git_branch:  # change to feature branch to test deployment
-    default: "kw-ie-keyboard-nav"
+    default: "cfignore-legacy-scripts"
     type: string
   prod_new_relic_app_id:
     default: "877570491"

frontend/src/pages/LegacyReport/index.js

@@ -5,6 +5,7 @@ import { Alert, Table } from '@trussworks/react-uswds';
 import { map } from 'lodash';
 
 import Container from '../../components/Container';
+import FileReviewItem from '../ActivityReport/Pages/Review/FileReviewItem';
 import { legacyReportById } from '../../fetchers/activityReports';
 import reportColumns from './reportColumns';
 
@@ -45,7 +46,7 @@ function LegacyReport({ match }) {
     );
   }
 
-  const { imported } = legacyReport;
+  const { imported, attachments } = legacyReport;
   const entries = map(reportColumns, (display, field) => {
     const value = imported[field];
     return {
@@ -57,9 +58,9 @@ function LegacyReport({ match }) {
 
   const tableEntries = entries.filter((item) => item.value).map(({ field, display, value }) => (
     <tr key={field}>
-      <td className="text-top">
+      <th scope="row" className="text-top">
         {display}
-      </td>
+      </th>
       <td>
         {value.split('\n').map((string) => <div key={string} className="margin-top-05">{string}</div>)}
       </td>
@@ -90,6 +91,27 @@ function LegacyReport({ match }) {
           </thead>
           <tbody>
             {tableEntries}
+            {attachments && attachments.length > 0
+              && (
+              <tr>
+                <th scope="row" className="text-top">Attachments</th>
+                <td>
+                  {attachments.map(({
+                    id,
+                    originalFileName,
+                    url: { url },
+                    status,
+                  }) => (
+                    <FileReviewItem
+                      key={id}
+                      filename={originalFileName}
+                      url={url}
+                      status={status}
+                    />
+                  ))}
+                </td>
+              </tr>
+              )}
           </tbody>
         </Table>
       </Container>

frontend/src/pages/LegacyReport/reportColumns.js

@@ -40,6 +40,7 @@ const reportFields = {
   additionalNotesForThisActivity: 'Additional notes for this activity',
   manager: 'Manager',
   managerApproval: 'Manager approval',
+  comments: 'Comments',
   created: 'Time Created',
   createdBy: 'Created By',
   modified: 'Time Modified',

package.json

@@ -65,7 +65,9 @@
     "import:goals:local": "./node_modules/.bin/babel-node ./src/tools/importTTAPlanGoals.js",
     "import:goals": "node ./build/server/tools/importTTAPlanGoals.js",
     "import:hses:local": "./node_modules/.bin/babel-node ./src/tools/importGrantGranteesCLI.js --skipdownload",
-    "import:hses": "node ./build/server/tools/importGrantGranteesCLI.js"
+    "import:hses": "node ./build/server/tools/importGrantGranteesCLI.js",
+    "reconcile:legacy": "node ./build/server/tools/reconcileLegacyReports.js",
+    "reconcile:legacy:local": "./node_modules/.bin/babel-node ./src/tools/reconcileLegacyReports.js"
   },
   "repository": {
     "type": "git",
@@ -105,7 +107,7 @@
         "statements": 90,
         "functions": 90,
         "branches": 81,
-        "lines": 90 
+        "lines": 90
       }
     }
   },

smartsheet_scripts/legacy_report_attachments/.ruby-version

@@ -0,0 +1 @@
+2.7.2

smartsheet_scripts/legacy_report_attachments/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gem "activesupport", "~> 6.1"
+gem "smartsheet", "~> 2.101"
+gem "rake", "~> 13.0"
+gem "faraday", "~> 1.3"
+gem "faraday_middleware", "~> 1.0"
+gem "faraday-cookie_jar"

smartsheet_scripts/legacy_report_attachments/Gemfile.lock

@@ -0,0 +1,57 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    awrence (1.2.1)
+    concurrent-ruby (1.1.8)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (1.3.0)
+      faraday-net_http (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
+      http-cookie (~> 1.0.0)
+    faraday-net_http (1.0.1)
+    faraday_middleware (1.0.0)
+      faraday (~> 1.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    i18n (1.8.9)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    multipart-post (2.1.1)
+    plissken (1.4.1)
+    rake (13.0.3)
+    ruby2_keywords (0.0.4)
+    smartsheet (2.101.1)
+      awrence (~> 1.0)
+      faraday (>= 0.13.1, < 2)
+      faraday_middleware (>= 0.10.0, < 2)
+      plissken (~> 1.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 6.1)
+  faraday (~> 1.3)
+  faraday-cookie_jar
+  faraday_middleware (~> 1.0)
+  rake (~> 13.0)
+  smartsheet (~> 2.101)
+
+BUNDLED WITH
+   2.1.4

smartsheet_scripts/legacy_report_attachments/README.md

@@ -0,0 +1,41 @@
+Legacy Report Attachments
+=========================
+
+Script to import comments and file attachments from Smartsheet ARs.
+
+Configuration
+-------------
+
+### Environment Variables
+
+To run, the following three ENV vars must be set:
+
+* `SMARTSHEET_API_TOKEN` api token for smartsheet, must have access to all 12 region AR sheets
+* `SMARTHUB_SESSION_COOKIE` the session cookie for a smarthub admin with read access to all 12 regions
+* `SMARTHUB_SESSION_SIG` the session.sig cooke for the same smarthub admin
+
+### smartsheet.yml
+
+This file contains the sheet ids for smartsheet activity report sheets
+
+### Attachment::SMARTHUB_URI_BASE
+
+This constant (set in `lib/attachment.rb`) is the base url of the smarthub
+
+Running
+-------
+
+* Ensure ruby 2.7.2 is installed
+* `bundle install`
+* Set ENV vars
+* `bundle exec rake run[REGION]` where `REGION` is the region number
+
+
+example: `bundle exec rake run[1]`
+
+
+Disclaimers
+-----------
+
+There are several attachments stored in onedrive that are behind additional login layers.
+These will need to be imported by hand

smartsheet_scripts/legacy_report_attachments/Rakefile

@@ -0,0 +1,6 @@
+require_relative "lib/attachment"
+
+desc "Transfer attachments by region"
+task :run, [:region] do |task, args|
+  Attachment.new(args[:region]).call
+end

smartsheet_scripts/legacy_report_attachments/lib/attachment.rb

@@ -0,0 +1,152 @@
+require "active_support/all"
+require "logger"
+require "smartsheet"
+require "faraday"
+require "faraday_middleware"
+require "faraday-cookie_jar"
+require "tempfile"
+require_relative "config"
+
+class Attachment
+
+  SMARTHUB_URI_BASE = "https://ttahub.ohs.acf.hhs.gov"
+
+  attr_reader :region, :client, :cookie
+  def initialize(region)
+    @region = region.to_s
+    # session cookies taken out of chrome dev tools for an active admin session
+    fail "Session cookie not provided" if ENV["SMARTHUB_SESSION_COOKIE"].blank? || ENV["SMARTHUB_SESSION_SIG"].blank?
+    @cookie = "session=#{ENV["SMARTHUB_SESSION_COOKIE"]}; session.sig=#{ENV["SMARTHUB_SESSION_SIG"]}"
+    token = ENV["SMARTSHEET_API_TOKEN"]
+    fail "No API Token provided" if token.blank?
+    logger = Logger.new($stdout)
+    logger.level = :warn
+    @client = Smartsheet::Client.new(
+      token: token,
+      logger: logger,
+      base_url: Smartsheet::Constants::GOV_API_URL
+    )
+  end
+
+  def call
+    sheet = client.sheets.get(sheet_id: sheet_id, params: {
+      include: "attachments,discussions",
+      level: "2",
+      columnIds: report_id_column_id
+    })
+    sheet[:rows].each do |row|
+      next if row[:discussions].blank? && row[:attachments].blank?
+      process_row(row)
+    end
+    sheet
+  end
+
+  def process_row(row)
+    legacy_id = row[:cells].first[:value].gsub(/^R14/, "R#{"%02d" % region}")
+    puts "Processing #{legacy_id}"
+    activity_report = if row[:discussions].present?
+      discussions = row[:discussions].flat_map { |disc| retrieve_discussion(disc[:id]) }.join("\n")
+      update_comments legacy_id, discussions
+    else
+      activity_report legacy_id
+    end
+    (row[:attachments] || []).each do |attachment|
+      process_attachment activity_report, attachment
+    end
+  end
+
+  def process_attachment(activity_report, attachment_meta)
+    if activity_report["attachments"].none? { |attach| attach["originalFileName"] == attachment_meta[:name] }
+      attach = client.sheets.attachments.get(sheet_id: sheet_id, attachment_id: attachment_meta[:id])
+      if attach[:url].blank?
+        fail "No URL to download file: #{attach.inspect}"
+      end
+      Tempfile.create do |file|
+        success = download_file(file, attach[:url])
+        if success
+          post_file(activity_report["id"], file, attachment_meta)
+        else
+          puts "Failed to download file for: #{activity_report["displayId"]}, #{attach.inspect}"
+        end
+      end
+    end
+  end
+
+  def update_comments(legacy_id, discussions)
+    response = put("/api/activity-reports/legacy/#{legacy_id}", {comments: discussions})
+    if response.success?
+      JSON.parse(response.body)
+    else
+      fail "#{response.status} #{response.body}"
+    end
+  end
+
+  def activity_report(legacy_id)
+    response = get("/api/activity-reports/legacy/#{legacy_id}")
+    if response.success?
+      JSON.parse(response.body)
+    else
+      fail "#{response.status} #{response.body}"
+    end
+  end
+
+  def download_file(file, url)
+    conn = Faraday.new(url) do |c|
+      c.use FaradayMiddleware::FollowRedirects
+      c.use :cookie_jar
+    end
+    response = conn.get
+    if response.success?
+      file.write(response.body)
+      file.flush
+      true
+    else
+      false
+    end
+  end
+
+  def post_file(report_id, file, attachment_meta)
+    filepart = Faraday::FilePart.new(file.path, attachment_meta[:mime_type], attachment_meta[:name])
+    params = {
+      "reportId": report_id,
+      file: filepart
+    }
+    multipart_faraday.post("/api/files", params)
+  end
+
+  def put(url, params)
+    faraday.put(url, params.to_json, "Content-Type" => "application/json")
+  end
+
+  def get(url)
+    faraday.get(url)
+  end
+
+  def multipart_faraday
+    @multipart_faraday ||= Faraday.new(url: SMARTHUB_URI_BASE, headers: {'Cookie' => cookie}) do |f|
+      f.request :multipart
+    end
+  end
+
+  def faraday
+    @faraday ||= Faraday.new(url: SMARTHUB_URI_BASE, headers: {'Cookie' => cookie})
+  end
+
+  def retrieve_discussion(disc_id)
+    client.sheets.discussions.get(sheet_id: sheet_id, discussion_id: disc_id)[:comments].map do |comment|
+      "#{comment[:created_by][:email]}: #{comment[:text]}"
+    end
+  end
+
+  def report_id_column_id
+    @report_id_column_id ||= sheet_columns.find { |c| c[:title] == "ReportID" }[:id]
+  end
+
+  def sheet_columns
+    @sheet_columns ||= client.sheets.columns.list(sheet_id: sheet_id)[:data]
+  end
+
+  def sheet_id
+    @sheet_id ||= Config.load[region][:ar_sheet_id]
+  end
+end

smartsheet_scripts/legacy_report_attachments/lib/config.rb

@@ -0,0 +1,17 @@
+require "yaml"
+require "erb"
+require "active_support/core_ext/hash"
+
+class Config
+  CONFIG_FILE = File.expand_path(File.join(__dir__, "..", "smartsheet.yml")).freeze
+
+  def self.load
+    @config ||= yaml.with_indifferent_access[:regions]
+  end
+
+  private
+
+  def self.yaml
+    @yaml ||= YAML.load(ERB.new(File.read(CONFIG_FILE)).result)
+  end
+end