04_upload_to_dataverse.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataverse API Token\n",
    "# https://dataverse.harvard.edu/dataverseuser.xhtml?selectTab=apiTokenTab\n",
    "host = 'https://dataverse.harvard.edu'\n",
    "token = '<API TOKEN>'\n",
    "doi = 'doi:10.7910/DVN/Q2KIES'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyDataverse.api import Api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyDataverse.models import Dataverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'OK'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "api = Api('https://dataverse.harvard.edu/', token)\n",
    "api.status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'OK',\n",
       " 'data': {'id': 6317607,\n",
       "  'identifier': 'DVN/Q2KIES',\n",
       "  'persistentUrl': 'https://doi.org/10.7910/DVN/Q2KIES',\n",
       "  'protocol': 'doi',\n",
       "  'authority': '10.7910',\n",
       "  'publisher': 'Harvard Dataverse',\n",
       "  'storageIdentifier': 's3://10.7910/DVN/Q2KIES',\n",
       "  'metadataLanguage': 'undefined',\n",
       "  'latestVersion': {'id': 328211,\n",
       "   'datasetId': 6317607,\n",
       "   'datasetPersistentId': 'doi:10.7910/DVN/Q2KIES',\n",
       "   'storageIdentifier': 's3://10.7910/DVN/Q2KIES',\n",
       "   'versionState': 'DRAFT',\n",
       "   'lastUpdateTime': '2022-06-06T01:03:36Z',\n",
       "   'createTime': '2022-06-06T01:03:36Z',\n",
       "   'license': {'name': 'CC0 1.0',\n",
       "    'uri': 'http://creativecommons.org/publicdomain/zero/1.0'},\n",
       "   'fileAccessRequest': False,\n",
       "   'metadataBlocks': {'citation': {'displayName': 'Citation Metadata',\n",
       "     'name': 'citation',\n",
       "     'fields': [{'typeName': 'title',\n",
       "       'multiple': False,\n",
       "       'typeClass': 'primitive',\n",
       "       'value': 'Fox News Transcripts'},\n",
       "      {'typeName': 'author',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'compound',\n",
       "       'value': [{'authorName': {'typeName': 'authorName',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'Sood, Gaurav'}}]},\n",
       "      {'typeName': 'datasetContact',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'compound',\n",
       "       'value': [{'datasetContactName': {'typeName': 'datasetContactName',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'Sood, Gaurav'},\n",
       "         'datasetContactEmail': {'typeName': 'datasetContactEmail',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'gsood07@gmail.com'}}]},\n",
       "      {'typeName': 'dsDescription',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'compound',\n",
       "       'value': [{'dsDescriptionValue': {'typeName': 'dsDescriptionValue',\n",
       "          'multiple': False,\n",
       "          'typeClass': 'primitive',\n",
       "          'value': 'Transcripts for Fox News'}}]},\n",
       "      {'typeName': 'subject',\n",
       "       'multiple': True,\n",
       "       'typeClass': 'controlledVocabulary',\n",
       "       'value': ['Social Sciences']},\n",
       "      {'typeName': 'depositor',\n",
       "       'multiple': False,\n",
       "       'typeClass': 'primitive',\n",
       "       'value': 'Sood, Gaurav'},\n",
       "      {'typeName': 'dateOfDeposit',\n",
       "       'multiple': False,\n",
       "       'typeClass': 'primitive',\n",
       "       'value': '2022-06-05'}]}},\n",
       "   'files': []}}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resp = api.get_dataset(doi)\n",
    "resp.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  668M    0     0  100  668M      0  34.6M  0:00:19  0:00:19 --:--:--     0"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'status': 'OK', 'data': {'files': [{'description': '', 'label': 'foxnews-transcript-html.tar.gz', 'restricted': False, 'version': 1, 'datasetVersionId': 328211, 'dataFile': {'id': 6318554, 'persistentId': '', 'pidURL': '', 'filename': 'foxnews-transcript-html.tar.gz', 'contentType': 'application/gzip', 'filesize': 701131684, 'description': '', 'storageIdentifier': 's3://dvn-cloud:1813794a4ba-0457a0af9970', 'rootDataFileId': -1, 'md5': 'ff691c081ab5c52179b03a2f053f3863', 'checksum': {'type': 'MD5', 'value': 'ff691c081ab5c52179b03a2f053f3863'}, 'creationDate': '2022-06-06'}}]}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100  668M  100   542  100  668M     27  33.4M  0:00:20  0:00:20 --:--:--   115\r",
      "100  668M  100   542  100  668M     27  33.4M  0:00:20  0:00:20 --:--:--   146\n"
     ]
    }
   ],
   "source": [
    "fn = 'foxnews-transcript-html.tar.gz'\n",
    "resp = api.upload_file(doi, fn)\n",
    "print(resp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100  149M    0     0  100  149M      0  30.6M  0:00:04  0:00:04 --:--:-- 30.6M"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'status': 'OK', 'data': {'files': [{'description': '', 'label': 'foxnews-transcript-text.tar.gz', 'restricted': False, 'version': 1, 'datasetVersionId': 328211, 'dataFile': {'id': 6318555, 'persistentId': '', 'pidURL': '', 'filename': 'foxnews-transcript-text.tar.gz', 'contentType': 'application/gzip', 'filesize': 157270297, 'description': '', 'storageIdentifier': 's3://dvn-cloud:1813794ca85-54d86ef2ee7c', 'rootDataFileId': -1, 'md5': 'c7972f954949e85b028996a12103878f', 'checksum': {'type': 'MD5', 'value': 'c7972f954949e85b028996a12103878f'}, 'creationDate': '2022-06-06'}}]}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100  149M  100   542  100  149M     95  26.5M  0:00:05  0:00:05 --:--:-- 25.8M\r",
      "100  149M  100   542  100  149M     95  26.5M  0:00:05  0:00:05 --:--:-- 18.1M\n"
     ]
    }
   ],
   "source": [
    "fn = 'foxnews-transcript-text.tar.gz'\n",
    "resp = api.upload_file(doi, fn)\n",
    "print(resp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      " 10 3170k    0     0   10  320k      0  1111k  0:00:02 --:--:--  0:00:02 1111k"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'status': 'OK', 'data': {'files': [{'description': '', 'label': 'foxnews-transcript.csv.gz', 'restricted': False, 'version': 1, 'datasetVersionId': 328211, 'dataFile': {'id': 6318556, 'persistentId': '', 'pidURL': '', 'filename': 'foxnews-transcript.csv.gz', 'contentType': 'application/gzip', 'filesize': 3246573, 'description': '', 'storageIdentifier': 's3://dvn-cloud:181379529dc-c56c923a0e51', 'rootDataFileId': -1, 'md5': 'd1c69c35e1c008a6e9c0d951f71aedf7', 'checksum': {'type': 'MD5', 'value': 'd1c69c35e1c008a6e9c0d951f71aedf7'}, 'creationDate': '2022-06-06'}}]}}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100 3171k  100   530  100 3170k    394  2360k  0:00:01  0:00:01 --:--:-- 2359k\r",
      "100 3171k  100   530  100 3170k    394  2359k  0:00:01  0:00:01 --:--:-- 2359k\n"
     ]
    }
   ],
   "source": [
    "fn = 'foxnews-transcript.csv.gz'\n",
    "resp = api.upload_file(doi, fn)\n",
    "print(resp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}