code-gov-github-stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook is meant to be executed using Python 3 and is a quick example on how to use the Github v4 API\n",
    "# Please fork this repo to add your own code.\n",
    "# If you'd like to make a contribution, please take a look at our README.md and submit a Pull Request.\n",
    "# For instructions on how to run this Jupyter Notebook please take a look at our README.md\n",
    "\n",
    "from graphqlclient import GraphQLClient\n",
    "import json\n",
    "import csv\n",
    "from datetime import datetime\n",
    "import time\n",
    "from dotenv import DotEnv\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = DotEnv()\n",
    "client = GraphQLClient('https://api.github.com/graphql')\n",
    "\n",
    "# You need a token to use the Github API in an efficient maner.\n",
    "# Please take a look at https://blog.github.com/2013-05-16-personal-api-tokens/ to see how to generate your own\n",
    "# Once you have your token please replace `your-token` with it.\n",
    "client.inject_token(f'bearer {env.get(\"GITHUB_TOKEN\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Our initial GraphQL query. Any updates done here need to also be made to the query_after query in the next cell.\n",
    "query_initial = '''query {\n",
    "    repositoryOwner(login: \"%s\") {\n",
    "      repositories(first: 100) {\n",
    "        edges {\n",
    "          node {\n",
    "            owner {\n",
    "              ... on User {\n",
    "                name\n",
    "                url\n",
    "                email\n",
    "              }\n",
    "              ... on Organization {\n",
    "                name\n",
    "                url\n",
    "                email\n",
    "              }\n",
    "            }\n",
    "            nameWithOwner\n",
    "            createdAt\n",
    "            forkCount\n",
    "            isFork\n",
    "            watchers {\n",
    "              totalCount\n",
    "            }\n",
    "            stargazers {\n",
    "              totalCount\n",
    "            }\n",
    "            pullRequests {\n",
    "              totalCount\n",
    "            }\n",
    "            issues(labels: [\"help wanted\", \"good first issue\"], first: 100) {\n",
    "              nodes {\n",
    "                locked\n",
    "                publishedAt\n",
    "                title\n",
    "                activeLockReason\n",
    "                lastEditedAt\n",
    "                url\n",
    "                body\n",
    "                bodyHTML\n",
    "                bodyText\n",
    "                participants {\n",
    "                  totalCount\n",
    "                }\n",
    "                comments {\n",
    "                  totalCount\n",
    "                }\n",
    "              }\n",
    "            }\n",
    "            languages(first: 20) {\n",
    "              nodes {\n",
    "                name\n",
    "              }\n",
    "            }\n",
    "          }\n",
    "        }\n",
    "        pageInfo {\n",
    "          hasNextPage\n",
    "          endCursor\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "    rateLimit {\n",
    "      limit\n",
    "      cost\n",
    "      remaining\n",
    "      resetAt\n",
    "    }\n",
    "  }'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GraphQL query that uses a cursor\n",
    "query_after = query = '''query {\n",
    "    repositoryOwner(login: \"%s\") {\n",
    "      repositories(first: 100, after: \"%s\") {\n",
    "        edges {\n",
    "          node {\n",
    "            owner {\n",
    "              ... on User {\n",
    "                name\n",
    "                url\n",
    "                email\n",
    "              }\n",
    "              ... on Organization {\n",
    "                name\n",
    "                url\n",
    "                email\n",
    "              }\n",
    "            }\n",
    "            nameWithOwner\n",
    "            createdAt\n",
    "            forkCount\n",
    "            isFork\n",
    "            watchers {\n",
    "              totalCount\n",
    "            }\n",
    "            stargazers {\n",
    "              totalCount\n",
    "            }\n",
    "            pullRequests {\n",
    "              totalCount\n",
    "            }\n",
    "            issues(labels: [\"help wanted\", \"good first issue\"], first: 100) {\n",
    "              nodes {\n",
    "                locked\n",
    "                publishedAt\n",
    "                title\n",
    "                activeLockReason\n",
    "                lastEditedAt\n",
    "                url\n",
    "                body\n",
    "                bodyHTML\n",
    "                bodyText\n",
    "                participants {\n",
    "                  totalCount\n",
    "                }\n",
    "                comments {\n",
    "                  totalCount\n",
    "                }\n",
    "              }\n",
    "            }\n",
    "            languages(first: 20) {\n",
    "              nodes {\n",
    "                name\n",
    "              }\n",
    "            }\n",
    "          }\n",
    "        }\n",
    "        pageInfo {\n",
    "          hasNextPage\n",
    "          endCursor\n",
    "        }\n",
    "      }\n",
    "    }\n",
    "    rateLimit {\n",
    "      limit\n",
    "      cost\n",
    "      remaining\n",
    "      resetAt\n",
    "    }\n",
    "  }'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This list is the U.S. Federal subset found here:\n",
    "# https://github.com/github/government.github.com/blob/gh-pages/_data/governments.yml\n",
    "owner_list = [\n",
    "    '18f',\n",
    "#     'adl-aicc',\n",
    "#     'adlnet',\n",
    "#     'afrl',\n",
    "#     'arcticlcc',\n",
    "#     'BBGInnovate',\n",
    "#     'bfelob',\n",
    "#     'blue-button',\n",
    "#     'businessusa',\n",
    "#     'CA-CST-Library',\n",
    "#     'CA-CST-SII',\n",
    "#     'ccmc',\n",
    "#     'CDCgov',\n",
    "#     'cfpb',\n",
    "#     'cmsgov',\n",
    "#     'commercedataservice',\n",
    "#     'commercegov',\n",
    "#     'defense-cyber-crime-center',\n",
    "#     'demand-driven-open-data',\n",
    "#     'department-of-veterans-affairs',\n",
    "#     'deptofdefense',\n",
    "#     'dhs-ncats',\n",
    "#     'didsr',\n",
    "#     'digital-analytics-program',\n",
    "#     'doecode',\n",
    "#     'EEOC',\n",
    "#     'energyapps',\n",
    "#     'erdc-cm',\n",
    "#     'eregs',\n",
    "#     'fcc',\n",
    "#     'fda',\n",
    "#     'fecgov',\n",
    "#     'Federal-Aviation-Administration',\n",
    "#     'federaltradecommission',\n",
    "#     'fedspendingtransparency',\n",
    "#     'GIST-ORNL',\n",
    "#     'globegit',\n",
    "#     'gopleader',\n",
    "#     'government-services',\n",
    "#     'GreatSmokyMountainsNationalPark',\n",
    "#     'gsa',\n",
    "#     'gsa-oes',\n",
    "#     'hhs',\n",
    "#     'HHS-AHRQ',\n",
    "#     'HHSDigitalMediaAPIPlatform',\n",
    "#     'HHSIDEAlab',\n",
    "#     'historyatstate',\n",
    "#     'IIP-Design',\n",
    "#     'IMDProjects',\n",
    "#     'imls',\n",
    "#     'informaticslab',\n",
    "#     'Innovation-Toolkit',\n",
    "#     'internationaltradeadministration',\n",
    "#     'ioos',\n",
    "#     'IRSgov',\n",
    "#     'keplergo',\n",
    "#     'libraryofcongress',\n",
    "#     'M-O-S-E-S',\n",
    "#     'mcc-gov',\n",
    "#     'missioncommand',\n",
    "    'nasa',\n",
    "    'nasa-develop',\n",
    "    'nasa-gibs',\n",
    "    'NASA-rdt',\n",
    "    'nasa-tournament-lab',\n",
    "    'nasaworldwind',\n",
    "#     'NationalGuard',\n",
    "#     'nationalparkservice',\n",
    "#     'NCRN',\n",
    "#     'NeoGeographyToolkit',\n",
    "#     'nesii',\n",
    "#     'ngds',\n",
    "#     'nhanes',\n",
    "#     'nidcd',\n",
    "#     'NIEM',\n",
    "#     'NMB-Dev',\n",
    "#     'NMML',\n",
    "#     'noaa-gfdl',\n",
    "#     'NOAA-ORR-ERD',\n",
    "#     'ntia',\n",
    "#     'ombegov',\n",
    "#     'Open-Sat',\n",
    "#     'opengovplatform',\n",
    "#     'ozoneplatform',\n",
    "#     'peacecorps',\n",
    "#     'petsc',\n",
    "#     'presidential-innovation-fellows',\n",
    "#     'project-open-data',\n",
    "#     'radiofreeasia',\n",
    "#     'redhawksdr',\n",
    "#     'regulationsgov',\n",
    "#     'sbstusa',\n",
    "#     'SelectUSA',\n",
    "#     'SERVIR',\n",
    "#     'SSAgov',\n",
    "#     'state-hiu',\n",
    "#     'sunpy',\n",
    "#     'us-bea',\n",
    "#     'US-CBP',\n",
    "#     'usagov',\n",
    "#     'usaid',\n",
    "#     'usajobs',\n",
    "#     'usasearch',\n",
    "#     'usbr',\n",
    "#     'uscensusbureau',\n",
    "#     'uscis',\n",
    "#     'usda',\n",
    "#     'usda-ars-agil',\n",
    "#     'USDA-ERS',\n",
    "#     'USDA-FSA',\n",
    "#     'usda-vs',\n",
    "#     'usdepartmentoflabor',\n",
    "#     'USDeptVeteransAffairs',\n",
    "#     'usdoj',\n",
    "#     'usds',\n",
    "#     'usdot-jpo-ode',\n",
    "#     'useda',\n",
    "#     'usepa',\n",
    "#     'USFWS',\n",
    "#     'usg-scope',\n",
    "#     'USGCRP',\n",
    "#     'usgpo',\n",
    "#     'usgs',\n",
    "#     'USGS-Astrogeology',\n",
    "#     'USGS-CIDA',\n",
    "#     'usgs-cmg',\n",
    "#     'usgs-eros',\n",
    "#     'USGS-OWI',\n",
    "#     'USGS-R',\n",
    "#     'USGS-WiM',\n",
    "#     'usindianaffairs',\n",
    "#     'usinterior',\n",
    "#     'usnationalarchives',\n",
    "#     'USPS',\n",
    "#     'USPTO',\n",
    "#     'USSBA',\n",
    "#     'usstatedept',\n",
    "#     'ustaxcourt',\n",
    "#     'VHAINNOVATIONS',\n",
    "#     'virtual-world-framework',\n",
    "#     'visionworkbench',\n",
    "#     'WFMRDA',\n",
    "#     'whitehouse',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_repos = []\n",
    "\n",
    "# Let's get all the data for the above owners\n",
    "\n",
    "for owner in owner_list:\n",
    "    hasNext = True\n",
    "    cursor = None\n",
    "    while hasNext:\n",
    "        \n",
    "        if cursor:\n",
    "            query = query_after % (owner, cursor)\n",
    "        else:\n",
    "            query = query_initial % owner\n",
    "\n",
    "        response = client.execute(query)\n",
    "        json_response = json.loads(response)\n",
    "        if json_response['data']:\n",
    "            for item in json_response['data']['repositoryOwner']['repositories']['edges']:\n",
    "                # These fields are later used to create the CSV\n",
    "                total_repos.append({\n",
    "                    'owner_name': item['node']['owner']['name'],\n",
    "                    'owner_email': item['node']['owner']['email'],\n",
    "                    'owner_url': item['node']['owner']['url'],\n",
    "                    'name_with_owner': item['node']['nameWithOwner'],\n",
    "                    'created_at': item['node']['createdAt'],\n",
    "                    'fork_count': item['node']['forkCount'],\n",
    "                    'is_fork': item['node']['isFork'],\n",
    "                    'watchers': item['node']['watchers']['totalCount'],\n",
    "                    'stargazers': item['node']['stargazers']['totalCount'],\n",
    "                    'pull_request': item['node']['pullRequests']['totalCount'],\n",
    "                    'issues': item['node']['issues'],\n",
    "                })\n",
    "\n",
    "            hasNext = json_response['data']['repositoryOwner']['repositories']['pageInfo']['hasNextPage']\n",
    "            cursor = json_response['data']['repositoryOwner']['repositories']['pageInfo']['endCursor']\n",
    "            remaining = json_response['data']['rateLimit']['remaining']\n",
    "            limit = json_response['data']['rateLimit']['limit']\n",
    "            percent_remaining = remaining / limit\n",
    "            reset_at = json_response['data']['rateLimit']['resetAt']\n",
    "            if percent_remaining < 0.15:\n",
    "                reset_at = datetime.strptime(reset_at, '%Y-%m-%dT%H:%M:%SZ')\n",
    "                current_time = datetime.now()\n",
    "                time_diff = current_time - reset_at\n",
    "                seconds = time_diff.total_seconds()\n",
    "                time.sleep(seconds)\n",
    "            else:\n",
    "                time.sleep(2)\n",
    "        else:\n",
    "            hasNext = false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same fields as above\n",
    "fieldnames = [\n",
    "    'owner_name',\n",
    "    'owner_email',\n",
    "    'owner_url',\n",
    "    'name_with_owner',\n",
    "    'created_at',\n",
    "    'fork_count',\n",
    "    'is_fork',\n",
    "    'watchers',\n",
    "    'stargazers',\n",
    "    'pull_request',\n",
    "    'issues',\n",
    "]\n",
    "with open('repos.csv', 'a') as csvfile:\n",
    "    csv_writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\n",
    "    csv_writer.writeheader()\n",
    "    csv_writer.writerows(total_repos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total NASA repos: 189\n",
      "Total 18F repos using filter function: 1053\n",
      "Total nasa repos using filter function: 189\n",
      "Total nasa_develop repos using filter function: 0\n",
      "Total nasa_gibs repos using filter function: 0\n",
      "Total NASA_rdt repos using filter function: 0\n",
      "Total nasa_tournament_lab repos using filter function: 0\n",
      "Total nasaworldwind repos using filter function: 0\n"
     ]
    }
   ],
   "source": [
    "# Example of what we can do with this.\n",
    "# Let's filter out only the repos that are from NASA\n",
    "# Then we'll filter out the repos that have issues\n",
    "# Then we'll print out the full name of the repositories\n",
    "\n",
    "# make a copy of our data set\n",
    "repos_to_play_with = total_repos\n",
    "\n",
    "# To filter our data set to only get the repos that are from NASA we can use a list comprehension.\n",
    "# More info here: https://docs.python.org/3/tutorial/datastructures.html#list-comprehensions\n",
    "nasa_repos = [repo for repo in repos_to_play_with if repo['owner_name'] == 'NASA']\n",
    "print(f'Total NASA repos: {len(nasa_repos)}')\n",
    "\n",
    "# Or for a more functional way (some say more \"correct\").\n",
    "# For filters more info here: https://docs.python.org/3/library/functions.html#filter\n",
    "# For lambda more info here: https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions\n",
    "nasa = list(filter(lambda repo: repo['owner_name'] == 'NASA', repos_to_play_with))\n",
    "print(f'Total nasa repos using filter function: {len(nasa)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Repo Full Name: nasa/openmct\n",
      "Repo Full Name: nasa/astrobot\n",
      "Repo Full Name: nasa/RHEAS\n",
      "Repo Full Name: nasa/podaacpy\n",
      "Repo Full Name: nasa/sitepod\n",
      "Repo Full Name: nasa/utm-apis\n"
     ]
    }
   ],
   "source": [
    "# Getting all repos with issues\n",
    "nasa_repos_with_issues = list(filter(lambda repo: len(repo['issues']['nodes']) > 0, nasa_repos))\n",
    "\n",
    "for repo in nasa_repos_with_issues:\n",
    "    print(f\"Repo Full Name: {repo['name_with_owner']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55af9aa3ff8f41b1a851821adefc280b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FigureCanvasNbAgg()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}