Pull request for #8, #9, #10: Incorporate new gslab_make and update template accordingly

.gitignore

@@ -1,4 +1,6 @@
 *config_user.yaml
+*external/
+*input/
 *.pyc
 *.lyx~
 *.lyx#

README.md

@@ -1,3 +1,67 @@
-# README 
+# README
 
+## Setup
+1. Install Python dependencies listed in `requirements.txt` using pip. One way to do this is to use the following bash command from the `setup` subdirectory:
+
+   ```
+   pip install --user -r requirements.txt
+   ```
 
+2. Run `setup_repository.py`. One way to do this is to use the following bash command from the `setup` subdirectory:
+   ```
+   python setup_repository.py
+   ```
+
+3. Install Stata dependencies using `setup_stata.do`. One way to do this is to use the following bash command from the `setup` subdirectory:
+   ```
+   stata-mp -e setup_stata.do
+   ```
+
+4. Install R dependencies using `setup_r.r`. One way to do this is to use the following bash command from the `setup` subdirector:y
+   ```
+   Rscript setup_r.r
+   ```
+
+## FAQ
+
+1. Help! I'm running into permission errors when trying to install Python dependencies!
+
+<br>
+
+![https://imgs.xkcd.com/comics/python_environment_2x.png](https://imgs.xkcd.com/comics/python_environment_2x.png){:height="50%" width="50%"}
+
+<br>
+
+The standard bash command for pip installing `requirements.txt` often runs into issues as depending on your Python environment, pip will attempt to install to a root directory (i.e., a directory that by default you should not have write permission)
+```
+pip install -r requirements.txtt
+```
+
+One way to get around this is to include `sudo` (or the Windows equivalent) in your bash command.
+```
+sudo pip install -r requirements.tx
+```
+
+However, we caution against this given the potential security risks. Instead, we recommend including the `--user` flag to your bash command.
+```
+pip install --user -r requirements.txt
+```
+
+The `--user` flag instructs pip to install to a local directory (i.e., a directory that by default you should have write permission). If you are still running into permission errors, we recommend the following diagnostic steps:
+
+   * Find the local directory that pip is attempting to install to. You can do this by either looking at the permission error message or using the following bash commands:
+   ```
+   python
+   import site
+   site.USER_SITE
+   ```
+
+   * If this is a directory that you should have write permission to but do not, use the `sudo chown` bash command (or the Windows equivalent) to get ownership.
+
+   * If this is a directory that you should not have write permission to, change your `PYTHONUSERBASE` environment variable to a diretory that you should and do have write permission to.
+
+If you are using Anaconda, we recommend using the following bash command:
+```
+conda install --file requirements.txt"
+```
+Note that you may run into issues if any of the Python depencencies are not available on the conda channels. If this is the case, revert back to using `pip`.

analysis/code/descriptive.py

@@ -2,13 +2,16 @@
 import matplotlib.pyplot as plt
 
 def main():
-    data  = np.genfromtxt('../input/data.txt', skip_header = 1)
+    data = np.genfromtxt('input/data_table.txt', skip_header = 1)
 
-    with open('../output/tables.txt', 'w') as f:
+    with open('output/tables.txt', 'w') as f:
         f.write('<tab:table>\n')
         f.write('%s\n%.3f\n%d\n%d' % (np.mean(data), np.std(data, ddof = 1), np.max(data), np.min(data)))
 
+    data = np.genfromtxt('input/data_graph.txt', skip_header = 1)
+
     plt.hist(data)
-    plt.savefig('../output/plot.eps')
+    plt.savefig('output/plot.eps')
 
-main()
+# EXECUTE
+main()

analysis/externals.txt

@@ -1 +1,3 @@
-# This file will contain paths to files that should be copied to /input/
+# Each line of instruction should contain the key for an external dependency listed in `config_user.yaml`
+
+# Key

analysis/inputs.txt

@@ -1 +1,6 @@
-# This file will contain paths to files that should be copied to /input/
+# Each line of instruction should contain a symlink and target delimited by a `|`
+# Lines beginning with # are ignored 
+
+# Symlink      | Target
+data_table.txt | {root}/data/output/data_table.txt
+data_graph.txt | {root}/data/output/data_graph.txt

analysis/log/link_heads.log

@@ -0,0 +1,28 @@
+File headers
+--------------------------------------------------------------------------------
+/Users/zong/template/data/output/data_table.txt
+--------------------------------------------------------------------------------
+x
+1
+2
+3
+4
+5
+6
+7
+8
+9
+--------------------------------------------------------------------------------
+/Users/zong/template/data/output/data_graph.txt
+--------------------------------------------------------------------------------
+x
+1
+2
+3
+4
+5
+6
+7
+8
+9
+--------------------------------------------------------------------------------

analysis/log/link_map.log

@@ -0,0 +1,3 @@
+symlink	target
+input/data_table.txt	/Users/zong/template/data/output/data_table.txt
+input/data_graph.txt	/Users/zong/template/data/output/data_graph.txt

analysis/log/link_stats.log

@@ -0,0 +1,3 @@
+file name	last modified	file size
+/Users/zong/template/data/output/data_table.txt	2018-12-19 19:19:34	1988897
+/Users/zong/template/data/output/data_graph.txt	2018-12-19 19:19:35	1988897

analysis/log/make.log

@@ -0,0 +1,13 @@
+--------------------------------------------------------------------------------
+Makelog started: 2018-12-19 11:19:54
+Working directory: /Users/zong/template/analysis
+--------------------------------------------------------------------------------
+Links successfully created!
+Links successfully created!
+Link logs successfully written!
+Executing: "python  "/Users/zong/template/analysis/code/descriptive.py" "
+Output logs successfully written!
+--------------------------------------------------------------------------------
+Makelog ended: 2018-12-19 11:19:57
+Working directory: /Users/zong/template/analysis
+--------------------------------------------------------------------------------

analysis/log/output_heads.log

@@ -0,0 +1,24 @@
+File headers
+--------------------------------------------------------------------------------
+/Users/zong/template/analysis/output/plot.eps
+--------------------------------------------------------------------------------
+%!PS-Adobe-3.0 EPSF-3.0
+%%Title: output/plot.eps
+%%Creator: matplotlib version 2.2.3, http://matplotlib.org/
+%%CreationDate: Wed Dec 19 11:19:57 2018
+%%Orientation: portrait
+%%BoundingBox: 75 223 536 568
+%%EndComments
+%%BeginProlog
+/mpldict 8 dict def
+mpldict begin
+--------------------------------------------------------------------------------
+/Users/zong/template/analysis/output/tables.txt
+--------------------------------------------------------------------------------
+<tab:table>
+150000.5
+86602.685
+300000
+1
+Head not readable or less than 10 lines
+--------------------------------------------------------------------------------

analysis/log/output_stats.log

@@ -0,0 +1,3 @@
+file name	last modified	file size
+/Users/zong/template/analysis/output/plot.eps	2018-12-19 19:19:57	12340
+/Users/zong/template/analysis/output/tables.txt	2018-12-19 19:19:56	39

analysis/make.py

@@ -1,20 +1,54 @@
-#! /usr/bin/env python
+# ENVIRONMENT
+import git
+import imp
+import os
+import yaml
 
-import shutil, os
-import gslab_make as gs
+ROOT = git.Repo('.', search_parent_directories = True).working_tree_dir
+f, path, desc = imp.find_module('gslab_make', [os.path.join(ROOT, 'lib')])
+gs = imp.load_module('gslab_make', f, path, desc)
 
-for dir in ['output', 'input', 'temp']:
-    shutil.rmtree(dir, ignore_errors='true')
-    os.mkdir(dir)
-os.chdir('code')
+PATHS = {
+    'config_user'     : '../config_user.yaml',
+    'input_dir'       : 'input', 
+    'external_dir'    : 'external',
+    'output_dir'      : 'output/',
+    'pdf_dir'         : 'output/',
+    'makelog'         : 'log/make.log',
+    'output_statslog' : 'log/output_stats.log',
+    'output_headslog' : 'log/output_heads.log', 
+    'link_maplog'     : 'log/link_map.log',
+    'link_statslog'   : 'log/link_stats.log',
+    'link_headslog'   : 'log/link_heads.log'
+}
 
-gs.start_make_logging()
+PATH_MAPPINGS = {
+    'root': ROOT
+}
 
-# GET EXTERNAL INPUT FILES
-os.symlink('../../data/output/data.txt', '../input/data.txt')
+# CONFIG USER
+config_user = yaml.load(open(PATHS['config_user'], 'rb'))
+if config_user['local']['executables']:
+    gs.private.metadata.default_executables[os.name].update(config_user['local']['executables']) 
+if config_user['external']:
+    PATH_MAPPINGS.update(config_user['external'])
+
+# START
+gs.remove_dir(['input', 'external'])
+gs.clear_dir(['output', 'log'])
+gs.start_makelog(PATHS)
+
+# GET INPUT FILES
+inputs = gs.create_input_links(PATHS, ['inputs.txt'], PATH_MAPPINGS)
+externals = gs.create_external_links(PATHS, ['externals.txt'], PATH_MAPPINGS)
+gs.write_link_logs(PATHS, inputs + externals)
 
 # RUN SCRIPTS
-gs.run_python(program = 'descriptive.py')
+gs.run_python(PATHS, program = 'code/descriptive.py')
+
+# LOG OUTPUTS
+gs.log_files_in_output(PATHS)
 
-gs.end_make_logging()
-raw_input('\n Press <Enter> to exit.')
+# END
+gs.end_makelog(PATHS)
+raw_input('\nPress <Enter> to exit.')

config.yaml

@@ -10,9 +10,9 @@ git_lfs_required: Yes
 #####################################################################
 # Other required software
 #####################################################################
+gslab_make_required: Yes
+
 software_required:
-  git_lfs: Yes
-  gslab_make: Yes
   r: Yes
   stata: Yes
   lyx: Yes

data/code/create_graph_data.do

@@ -0,0 +1,11 @@
+clear all
+set more off
+
+program main
+    set obs 300000
+    gen x = _n
+    export delimited "output/data_graph.txt", delimiter("|") replace
+end
+
+* EXECUTE
+main

data/code/create_table_data.r

@@ -0,0 +1,9 @@
+library(yaml)
+
+main <- function() {
+  x <- 1:300000
+  write.table(x, "output/data_table.txt", row.names = FALSE, col.names = TRUE, quote = FALSE)
+}
+
+# EXECUTE
+main()

data/externals.txt

@@ -1 +1,3 @@
-# This file will contain paths to files that should be copied to /input/
+# Each line of instruction should contain the key for an external dependency listed in `config_user.yaml`
+
+# Key

data/inputs.txt

@@ -1 +1,4 @@
-# This file will contain paths to files that should be copied to /input/
+# Each line of instruction should contain a symlink and target delimited by a `|`
+# Lines beginning with # are ignored 
+
+# Symlink | Target

data/log/link_heads.log

@@ -0,0 +1,2 @@
+File headers
+--------------------------------------------------------------------------------

data/log/link_map.log

@@ -0,0 +1 @@
+symlink	target

data/log/link_stats.log

@@ -0,0 +1 @@
+file name	last modified	file size