Merge pull request #2715 from gojomo/py38-plus-build-tuning

closes #2713

.travis.yml

@@ -17,11 +17,10 @@ matrix:
     - python: '3.6'
       env: TOXENV="flake8,flake8-docs"
 
-    - python: '3.5'
-      env: TOXENV="py35-linux"
-
-    - python: '3.6'
-      env: TOXENV="py36-linux"
+    - python: '3.8'
+      env:
+        - TOXENV="py38-linux"
+      dist: bionic
 
     - python: '3.7'
       env:
@@ -30,8 +29,23 @@ matrix:
       dist: xenial
       sudo: true
 
+    - python: '3.6'
+      env: TOXENV="py36-linux"
+
 
 install:
   - pip install tox
+  - sudo apt-get install -y gdb  # install gdb
+
+
+before_script:
+  - ulimit -c unlimited -S       # enable core dumps
+
 
 script: tox -vv
+
+
+after_failure:
+  - pwd
+  - COREFILE=$(find . -maxdepth 1 -name "core*" | head -n 1) # find core file
+  - if [[ -f "$COREFILE" ]]; then EXECFILE=$(gdb -c "$COREFILE" -batch | grep "Core was generated" | tr -d "\`" | cut -d' ' -f5); file "$COREFILE"; gdb -c "$COREFILE" "$EXECFILE" -x continuous_integration/debug.gdb -batch; fi

appveyor.yml

@@ -14,21 +14,23 @@ environment:
     TOX_PARALLEL_NO_SPINNER: 1
 
   matrix:
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.2"
+# Python 3.8 builds on Appveyor/Windows failing; commented-out for now pending resolution
+# see comment at <https://github.com/RaRe-Technologies/gensim/pull/2715#issuecomment-569457589>
+#    - PYTHON: "C:\\Python38-x64"
+#      PYTHON_VERSION: "3.8.1"
+#      PYTHON_ARCH: "64"
+#      TOXENV: "py38-win"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7.6"
       PYTHON_ARCH: "64"
-      TOXENV: "py35-win"
+      TOXENV: "py37-win"
 
     - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "3.6.0"
+      PYTHON_VERSION: "3.6.10"
       PYTHON_ARCH: "64"
       TOXENV: "py36-win"
 
-    - PYTHON: "C:\\Python37-x64"
-      PYTHON_VERSION: "3.7.0"
-      PYTHON_ARCH: "64"
-      TOXENV: "py37-win"
-
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
   - "ECHO \"%APPVEYOR_SCHEDULED_BUILD%\""
@@ -50,9 +52,15 @@ install:
   - "powershell ./continuous_integration/appveyor/install.ps1"
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "python -m pip install -U pip tox"
-  - "pip --version"
 
-  # Check that we have the expected version and architecture for Python
+  # Next line only to demo that py3.8-on-Appveyor *could* install Cython from wheel just fine,
+  # despite mysterious following attempt/failure to build-and-use Cython on that one Appveyor config.
+  # Delete when py3.8-on-Appveyor starts working normally,
+  # see comment at <https://github.com/RaRe-Technologies/gensim/pull/2715#issuecomment-569457589>
+  - "python -m pip install Cython==0.29.14 numpy==1.18.0"
+
+  # Check that we have the expected versions and architecture
+  - "pip --version"
   - "python --version"
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 

continuous_integration/debug.gdb

@@ -0,0 +1,21 @@
+# commands to run on CI machine in event of testing core-dump
+
+set trace-commands on
+
+thread apply all bt
+
+f
+info args
+info locals
+
+up
+
+f
+info args
+info locals
+
+up
+
+f
+info args
+info locals

gensim/corpora/sharded_corpus.py

@@ -280,12 +280,12 @@ def init_shards(self, output_prefix, corpus, shardsize=4096, dtype=_default_dtyp
         self.dim = proposed_dim
         self.offsets = [0]
 
-        start_time = time.clock()
+        start_time = time.perf_counter()
 
         logger.info('Running init from corpus.')
 
         for n, doc_chunk in enumerate(gensim.utils.grouper(corpus, chunksize=shardsize)):
-            logger.info('Chunk no. %d at %f s', n, time.clock() - start_time)
+            logger.info('Chunk no. %d at %f s', n, time.perf_counter() - start_time)
 
             current_shard = numpy.zeros((len(doc_chunk), self.dim), dtype=dtype)
             logger.debug('Current chunk dimension: %d x %d', len(doc_chunk), self.dim)
@@ -300,7 +300,7 @@ def init_shards(self, output_prefix, corpus, shardsize=4096, dtype=_default_dtyp
 
             self.save_shard(current_shard)
 
-        end_time = time.clock()
+        end_time = time.perf_counter()
         logger.info('Built %d shards in %f s.', self.n_shards, end_time - start_time)
 
     def init_by_clone(self):

gensim/models/hdpmodel.py

@@ -464,10 +464,7 @@ def update(self, corpus):
         """
         save_freq = max(1, int(10000 / self.chunksize))  # save every 10k docs, roughly
         chunks_processed = 0
-        try:
-            start_time = time.time()
-        except AttributeError:
-            start_time = time.clock()
+        start_time = time.perf_counter()
 
         while True:
             for chunk in utils.grouper(corpus, self.chunksize):
@@ -511,16 +508,12 @@ def update_finished(self, start_time, chunks_processed, docs_processed):
             If True - model is updated, False otherwise.
 
         """
-        try:
-            start_time = time.time()
-        except AttributeError:
-            start_time = time.clock()
         return (
             # chunk limit reached
             (self.max_chunks and chunks_processed == self.max_chunks)
 
             # time limit reached
-            or (self.max_time and start_time - start_time > self.max_time)
+            or (self.max_time and time.perf_counter() - start_time > self.max_time)
 
             # no limits and whole corpus has been processed once
             or (not self.max_chunks and not self.max_time and docs_processed >= self.m_D))

setup.py

@@ -321,7 +321,7 @@ def run(self):
 # to build with any sane version of Cython, so we should update this pin
 # periodically.
 #
-CYTHON_STR = 'Cython==0.29.3'
+CYTHON_STR = 'Cython==0.29.14'
 
 install_requires = [
     NUMPY_STR,

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 2.0
-envlist = {py35,py36,py37}-{win,linux}, flake8, docs, docs-upload, download-wheels, upload-wheels, test-pypi
+envlist = {py36,py37,py38}-{win,linux}, flake8, docs, docs-upload, download-wheels, upload-wheels, test-pypi
 skipsdist = True
 platform = linux: linux
            win: win64
@@ -64,7 +64,7 @@ commands = flake8-rst gensim/ docs/ {posargs}
 basepython = python3
 recreate = True
 
-deps = numpy==1.14.5
+deps = numpy
 commands = python setup.py build_ext --inplace