Include attchment transition script in full repo

smartsheet_scripts/legacy_report_attachments/.ruby-version

@@ -0,0 +1 @@
+2.7.2

smartsheet_scripts/legacy_report_attachments/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gem "activesupport", "~> 6.1"
+gem "smartsheet", "~> 2.101"
+gem "rake", "~> 13.0"
+gem "faraday", "~> 1.3"
+gem "faraday_middleware", "~> 1.0"
+gem "faraday-cookie_jar"

smartsheet_scripts/legacy_report_attachments/Gemfile.lock

@@ -0,0 +1,57 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    awrence (1.2.1)
+    concurrent-ruby (1.1.8)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (1.3.0)
+      faraday-net_http (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
+      http-cookie (~> 1.0.0)
+    faraday-net_http (1.0.1)
+    faraday_middleware (1.0.0)
+      faraday (~> 1.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    i18n (1.8.9)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    multipart-post (2.1.1)
+    plissken (1.4.1)
+    rake (13.0.3)
+    ruby2_keywords (0.0.4)
+    smartsheet (2.101.1)
+      awrence (~> 1.0)
+      faraday (>= 0.13.1, < 2)
+      faraday_middleware (>= 0.10.0, < 2)
+      plissken (~> 1.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport (~> 6.1)
+  faraday (~> 1.3)
+  faraday-cookie_jar
+  faraday_middleware (~> 1.0)
+  rake (~> 13.0)
+  smartsheet (~> 2.101)
+
+BUNDLED WITH
+   2.1.4

smartsheet_scripts/legacy_report_attachments/README.md

@@ -0,0 +1,41 @@
+Legacy Report Attachments
+=========================
+
+Script to import comments and file attachments from Smartsheet ARs.
+
+Configuration
+-------------
+
+### Environment Variables
+
+To run, the following three ENV vars must be set:
+
+* `SMARTSHEET_API_TOKEN` api token for smartsheet, must have access to all 12 region AR sheets
+* `SMARTHUB_SESSION_COOKIE` the session cookie for a smarthub admin with read access to all 12 regions
+* `SMARTHUB_SESSION_SIG` the session.sig cooke for the same smarthub admin
+
+### smartsheet.yml
+
+This file contains the sheet ids for smartsheet activity report sheets
+
+### Attachment::SMARTHUB_URI_BASE
+
+This constant (set in `lib/attachment.rb`) is the base url of the smarthub
+
+Running
+-------
+
+* Ensure ruby 2.7.2 is installed
+* `bundle install`
+* Set ENV vars
+* `bundle exec rake run[REGION]` where `REGION` is the region number
+
+
+example: `bundle exec rake run[1]`
+
+
+Disclaimers
+-----------
+
+There are several attachments stored in onedrive that are behind additional login layers.
+These will need to be imported by hand

smartsheet_scripts/legacy_report_attachments/Rakefile

@@ -0,0 +1,6 @@
+require_relative "lib/attachment"
+
+desc "Transfer attachments by region"
+task :run, [:region] do |task, args|
+  Attachment.new(args[:region]).call
+end

smartsheet_scripts/legacy_report_attachments/lib/attachment.rb

@@ -0,0 +1,152 @@
+require "active_support/all"
+require "logger"
+require "smartsheet"
+require "faraday"
+require "faraday_middleware"
+require "faraday-cookie_jar"
+require "tempfile"
+require_relative "config"
+
+class Attachment
+
+  SMARTHUB_URI_BASE = "https://ttahub.ohs.acf.hhs.gov"
+
+  attr_reader :region, :client, :cookie
+  def initialize(region)
+    @region = region.to_s
+    # session cookies taken out of chrome dev tools for an active admin session
+    fail "Session cookie not provided" if ENV["SMARTHUB_SESSION_COOKIE"].blank? || ENV["SMARTHUB_SESSION_SIG"].blank?
+    @cookie = "session=#{ENV["SMARTHUB_SESSION_COOKIE"]}; session.sig=#{ENV["SMARTHUB_SESSION_SIG"]}"
+    token = ENV["SMARTSHEET_API_TOKEN"]
+    fail "No API Token provided" if token.blank?
+    logger = Logger.new($stdout)
+    logger.level = :warn
+    @client = Smartsheet::Client.new(
+      token: token,
+      logger: logger,
+      base_url: Smartsheet::Constants::GOV_API_URL
+    )
+  end
+
+  def call
+    sheet = client.sheets.get(sheet_id: sheet_id, params: {
+      include: "attachments,discussions",
+      level: "2",
+      columnIds: report_id_column_id
+    })
+    sheet[:rows].each do |row|
+      next if row[:discussions].blank? && row[:attachments].blank?
+      process_row(row)
+    end
+    sheet
+  end
+
+  def process_row(row)
+    legacy_id = row[:cells].first[:value].gsub(/^R14/, "R#{"%02d" % region}")
+    puts "Processing #{legacy_id}"
+    activity_report = if row[:discussions].present?
+      discussions = row[:discussions].flat_map { |disc| retrieve_discussion(disc[:id]) }.join("\n")
+      update_comments legacy_id, discussions
+    else
+      activity_report legacy_id
+    end
+    (row[:attachments] || []).each do |attachment|
+      process_attachment activity_report, attachment
+    end
+  end
+
+  def process_attachment(activity_report, attachment_meta)
+    if activity_report["attachments"].none? { |attach| attach["originalFileName"] == attachment_meta[:name] }
+      attach = client.sheets.attachments.get(sheet_id: sheet_id, attachment_id: attachment_meta[:id])
+      if attach[:url].blank?
+        fail "No URL to download file: #{attach.inspect}"
+      end
+      Tempfile.create do |file|
+        success = download_file(file, attach[:url])
+        if success
+          post_file(activity_report["id"], file, attachment_meta)
+        else
+          puts "Failed to download file for: #{activity_report["displayId"]}, #{attach.inspect}"
+        end
+      end
+    end
+  end
+
+  def update_comments(legacy_id, discussions)
+    response = put("/api/activity-reports/legacy/#{legacy_id}", {comments: discussions})
+    if response.success?
+      JSON.parse(response.body)
+    else
+      fail "#{response.status} #{response.body}"
+    end
+  end
+
+  def activity_report(legacy_id)
+    response = get("/api/activity-reports/legacy/#{legacy_id}")
+    if response.success?
+      JSON.parse(response.body)
+    else
+      fail "#{response.status} #{response.body}"
+    end
+  end
+
+  def download_file(file, url)
+    conn = Faraday.new(url) do |c|
+      c.use FaradayMiddleware::FollowRedirects
+      c.use :cookie_jar
+    end
+    response = conn.get
+    if response.success?
+      file.write(response.body)
+      file.flush
+      true
+    else
+      false
+    end
+  end
+
+  def post_file(report_id, file, attachment_meta)
+    filepart = Faraday::FilePart.new(file.path, attachment_meta[:mime_type], attachment_meta[:name])
+    params = {
+      "reportId": report_id,
+      file: filepart
+    }
+    multipart_faraday.post("/api/files", params)
+  end
+
+  def put(url, params)
+    faraday.put(url, params.to_json, "Content-Type" => "application/json")
+  end
+
+  def get(url)
+    faraday.get(url)
+  end
+
+  def multipart_faraday
+    @multipart_faraday ||= Faraday.new(url: SMARTHUB_URI_BASE, headers: {'Cookie' => cookie}) do |f|
+      f.request :multipart
+    end
+  end
+
+  def faraday
+    @faraday ||= Faraday.new(url: SMARTHUB_URI_BASE, headers: {'Cookie' => cookie})
+  end
+
+  def retrieve_discussion(disc_id)
+    client.sheets.discussions.get(sheet_id: sheet_id, discussion_id: disc_id)[:comments].map do |comment|
+      "#{comment[:created_by][:email]}: #{comment[:text]}"
+    end
+  end
+
+  def report_id_column_id
+    @report_id_column_id ||= sheet_columns.find { |c| c[:title] == "ReportID" }[:id]
+  end
+
+  def sheet_columns
+    @sheet_columns ||= client.sheets.columns.list(sheet_id: sheet_id)[:data]
+  end
+
+  def sheet_id
+    @sheet_id ||= Config.load[region][:ar_sheet_id]
+  end
+end

smartsheet_scripts/legacy_report_attachments/lib/config.rb

@@ -0,0 +1,17 @@
+require "yaml"
+require "erb"
+require "active_support/core_ext/hash"
+
+class Config
+  CONFIG_FILE = File.expand_path(File.join(__dir__, "..", "smartsheet.yml")).freeze
+
+  def self.load
+    @config ||= yaml.with_indifferent_access[:regions]
+  end
+
+  private
+
+  def self.yaml
+    @yaml ||= YAML.load(ERB.new(File.read(CONFIG_FILE)).result)
+  end
+end

smartsheet_scripts/legacy_report_attachments/smartsheet.yml

@@ -0,0 +1,25 @@
+regions:
+  "1":
+    ar_sheet_id: "1515864581675124"
+  "2":
+    ar_sheet_id: "5139786187348084"
+  "3":
+    ar_sheet_id: "2184092773455988"
+  "4":
+    ar_sheet_id: "2458901960923252"
+  "5":
+    ar_sheet_id: "1333002054080628"
+  "6":
+    ar_sheet_id: "6793245517092980"
+  "7":
+    ar_sheet_id: "5175863912634484"
+  "8":
+    ar_sheet_id: "4964482802194548"
+  "9":
+    ar_sheet_id: "2924064098949236"
+  "10":
+    ar_sheet_id: "7427663726319732"
+  "11":
+    ar_sheet_id: "354711583266932"
+  "12":
+    ar_sheet_id: "2712682988509300"