Revert "PLAT-2060 Use pip-tools to manage requirements files"

Makefile

@@ -1,14 +1,9 @@
 # Do things in edx-platform
-.PHONY: clean extract_translations help pull_translations push_translations requirements upgrade
 
 # Careful with mktemp syntax: it has to work on Mac and Ubuntu, which have differences.
 PRIVATE_FILES := $(shell mktemp -u /tmp/private_files.XXXXXX)
 
-help: ## display this help message
-	@echo "Please use \`make <target>' where <target> is one of"
-	@grep '^[a-zA-Z]' $(MAKEFILE_LIST) | sort | awk -F ':.*?## ' 'NF==2 {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
-
-clean: ## archive and delete most git-ignored files
+clean:
 	# Remove all the git-ignored stuff, but save and restore things marked
 	# by start-noclean/end-noclean. Include Makefile in the tarball so that
 	# there's always at least one file even if there are no private files.
@@ -18,13 +13,16 @@ clean: ## archive and delete most git-ignored files
 	tar xf $(PRIVATE_FILES)
 	rm $(PRIVATE_FILES)
 
-extract_translations: ## extract localizable strings from sources
+extract_translations:
+	# Extract localizable strings from sources
 	i18n_tool extract -vv
 
-push_translations: ## push source strings to Transifex for translation
+push_translations:
+	# Push source strings to Transifex for translation
 	i18n_tool transifex push
 
-pull_translations: ## pull translations from Transifex
+pull_translations:
+	## Pull translations from Transifex
 	git clean -fdX conf/locale
 	i18n_tool transifex pull
 	i18n_tool extract
@@ -34,32 +32,3 @@ pull_translations: ## pull translations from Transifex
 	git clean -fdX conf/locale/rtl
 	git clean -fdX conf/locale/eo
 	i18n_tool validate
-
-requirements: ## install development environment requirements
-	pip install -qr requirements/edx/development.txt --exists-action w
-
-upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints
-	pip install -qr requirements/edx/pip-tools.txt
-	# Make sure to compile files after any other files they include!
-	pip-compile --upgrade -o requirements/edx/pip-tools.txt requirements/edx/pip-tools.in
-	pip-compile --upgrade -o requirements/edx/coverage.txt requirements/edx/coverage.in
-	pip-compile --upgrade -o requirements/edx/paver.txt requirements/edx/paver.in
-	pip-compile --upgrade -o requirements/edx-sandbox/shared.txt requirements/edx-sandbox/shared.in
-	pip-compile --upgrade -o requirements/edx-sandbox/base.txt requirements/edx-sandbox/base.in
-	pip-compile --upgrade -o requirements/edx/base.txt requirements/edx/base.in
-	pip-compile --upgrade -o requirements/edx/testing.txt requirements/edx/testing.in
-	pip-compile --upgrade -o requirements/edx/development.txt requirements/edx/development.in
-	# Post process all of the files generated above to work around open pip-tools issues
-	scripts/post-pip-compile.sh \
-        requirements/edx/pip-tools.txt \
-	    requirements/edx/coverage.txt \
-	    requirements/edx/paver.txt \
-	    requirements/edx-sandbox/shared.txt \
-	    requirements/edx-sandbox/base.txt \
-	    requirements/edx/base.txt \
-	    requirements/edx/testing.txt \
-	    requirements/edx/development.txt
-	# Let tox control the Django version for tests
-	grep "^django==" requirements/edx/base.txt > requirements/edx/django.txt
-	sed '/^[dD]jango==/d' requirements/edx/testing.txt > requirements/edx/testing.tmp
-	mv requirements/edx/testing.tmp requirements/edx/testing.txt

circle.yml

@@ -12,14 +12,26 @@ dependencies:
     - npm install
 
     - pip install setuptools
+    - pip install --exists-action w -r requirements/edx/paver.txt
 
     # Mirror what paver install_prereqs does.
     # After a successful build, CircleCI will
     # cache the virtualenv at that state, so that
     # the next build will not need to install them
     # from scratch again.
     - pip install --exists-action w -r requirements/edx/pre.txt
+    - pip install --exists-action w -r requirements/edx/github.txt
+    - pip install --exists-action w -r requirements/edx/local.txt
+
+    # HACK: within base.txt stevedore had a
+    # dependency on a version range of pbr.
+    # Install a version which falls within that range.
+    - pip install  --exists-action w pbr==0.9.0
+    - pip install --exists-action w -r requirements/edx/django.txt
+    - pip install --exists-action w -r requirements/edx/base.txt
+    - pip install --exists-action w -r requirements/edx/paver.txt
     - pip install --exists-action w -r requirements/edx/testing.txt
+    - if [ -e requirements/edx/post.txt ]; then pip install --exists-action w -r requirements/edx/post.txt ; fi
 
     - pip install coveralls==1.0
 

common/lib/capa/capa/safe_exec/README.rst

@@ -19,6 +19,8 @@ __ https://github.com/edx/codejail/blob/master/README.rst
    need to install the requirements from requirements/edx-sandbox::
 
     $ pip install -r requirements/edx-sandbox/base.txt
+    $ pip install -r requirements/edx-sandbox/local.txt
+    $ pip install -r requirements/edx-sandbox/post.txt
 
 2. At the instruction to create the AppArmor profile, you'll need a line in
    the profile for the sandbox packages.  <EDXPLATFORM> is the full path to

common/lib/chem/setup.py

@@ -8,6 +8,6 @@
         "pyparsing==2.2.0",
         "numpy==1.6.2",
         "scipy==0.14.0",
-        "nltk",
+        "nltk==3.2.5",
     ],
 )

openedx.yaml

@@ -3,5 +3,5 @@
 
 nick: edx
 oeps: {}
-openedx-release: {ref: release}
+openedx-release: {ref: release, requirements: requirements/edx/github.txt}
 track-pulls: true

openedx/core/djangoapps/user_api/tests/test_views.py

@@ -2590,8 +2590,7 @@ def _assert_time_zone_is_valid(self, time_zone_info):
         self.assertIn(time_zone_name, common_timezones_set)
         self.assertEqual(time_zone_info['description'], get_display_time_zone(time_zone_name))
 
-    # The time zones count may need to change each time we upgrade pytz
-    @ddt.data((ALL_TIME_ZONES_URI, 439),
+    @ddt.data((ALL_TIME_ZONES_URI, 436),
               (COUNTRY_TIME_ZONES_URI, 28))
     @ddt.unpack
     def test_get_basic(self, country_uri, expected_count):

pavelib/prereqs.py

@@ -21,10 +21,20 @@
 # If you make any changes to this list you also need to make
 # a corresponding change to circle.yml, which is how the python
 # prerequisites are installed for builds on circleci.com
+PYTHON_REQ_FILES = [
+    'requirements/edx/pre.txt',
+    'requirements/edx/github.txt',
+    'requirements/edx/local.txt',
+    'requirements/edx/django.txt',
+    'requirements/edx/base.txt',
+    'requirements/edx/paver.txt',
+    'requirements/edx/development.txt',
+    'requirements/edx/testing.txt',
+    'requirements/edx/post.txt',
+]
 if 'TOXENV' in os.environ:
-    PYTHON_REQ_FILES = ['requirements/edx/testing.txt']
-else:
-    PYTHON_REQ_FILES = ['requirements/edx/development.txt']
+    # Let tox manage the Django version
+    PYTHON_REQ_FILES.remove('requirements/edx/django.txt')
 
 # Developers can have private requirements, for local copies of github repos,
 # or favorite debugging tools, etc.

pavement.py

@@ -6,7 +6,7 @@
 # In local dev, we usually don't need to do this, because Python
 # automatically puts the current working directory on the system path.
 # In Jenkins, however, we have multiple copies of the edx-platform repo,
-# each of which run "pip install -e ." (as part of requirements/edx/local.in)
+# each of which run "pip install -e ." (as part of requirements/edx/local.txt)
 # Until we re-run pip install, the other copies of edx-platform could
 # take precedence, leading to some very strange results.
 sys.path.insert(0, os.path.dirname(__file__))

requirements/edx-sandbox/base.txt

@@ -1,33 +1,16 @@
+# DON'T JUST ADD NEW DEPENDENCIES!!!
 #
-# This file is autogenerated by pip-compile
-# To update, run:
-#
-#    make upgrade
-#
+# If you open a pull request that adds a new dependency, you should notify:
+#   * @edx/ospr - to check licensing
+#   * @edx/devops - to check system requirements
+
+# Pin packaging tools the same as edxapp.  Keep them in sync for our sanity.
+setuptools==37.0.0
+pip==9.0.1
 
--e common/lib/calc
--e common/lib/chem
--e common/lib/sandbox-packages
--e common/lib/symmath
-asn1crypto==0.24.0
-backports-abc==0.5        # via tornado
-cffi==1.11.5
-cryptography==2.1.4
-enum34==1.1.6
-futures==3.2.0            # via tornado
-idna==2.6
-ipaddress==1.0.19
-lxml==3.8.0
-matplotlib==1.3.1
-networkx==1.7
 nltk==3.2.5
-nose==1.3.7               # via matplotlib
 numpy==1.6.2
-pycparser==2.18
-pyparsing==2.0.7
-python-dateutil==2.7.2    # via matplotlib
-scipy==0.14.0
-singledispatch==3.4.0.3   # via tornado
-six==1.11.0
+networkx==1.7
 sympy==0.7.1
-tornado==5.0.2            # via matplotlib
+pyparsing==2.2.0
+cryptography==1.9

requirements/edx-sandbox/local.txt

@@ -1,3 +1,13 @@
-# Placeholder for code which hasn't yet been updated to no longer use this file.
-# The requirements which were here have been moved to shared.in, and are now
-# incorporated into base.txt
+# DON'T JUST ADD NEW DEPENDENCIES!!!
+#
+# If you open a pull request that adds a new dependency, you should notify:
+#   * @edx/ospr - to check licensing
+#   * @edx/devops - to check system requirements
+
+# Install these packages from the edx-platform working tree
+# NOTE: if you change code in these packages, you MUST change the version
+# number in its setup.py or the code WILL NOT be installed during deploy.
+common/lib/calc
+common/lib/chem
+common/lib/sandbox-packages
+common/lib/symmath

requirements/edx-sandbox/post.txt

@@ -1,3 +1,11 @@
-# Placeholder for code which hasn't yet been updated to no longer use this file.
-# The requirements which were here have been moved to shared.in and base.in,
-# and are now incorporated into base.txt
+# DON'T JUST ADD NEW DEPENDENCIES!!!
+#
+# If you open a pull request that adds a new dependency, you should notify:
+#   * @edx/ospr - to check licensing
+#   * @edx/devops - to check system requirements
+
+# Packages to install in the Python sandbox for secured execution.
+scipy==0.14.0
+# lxml is also in requirements/edx/base.txt
+lxml==3.8.0
+matplotlib==1.3.1