Fix of windows compatibility issues

- binary files need to be opened with 'rb' or 'wb' flag
- leveldb package is not available for windows - display warning instead of crashing
- adapted find_executable with fallback for windows ".exe" extension and let the version check pass
- normalization of path separators to linux style
- turn off fcntl on windows
- Making tests pass and support of file upload
- portable cudart localization

digits/config/caffe_option.py

@@ -65,6 +65,8 @@ def validate(cls, value):
  if value == '<PATHS>':
  # Find the executable
  executable = cls.find_executable('caffe')
+ if not executable:
+ executable = cls.find_executable('caffe.exe')
  if not executable:
  raise config_option.BadValue('caffe binary not found in PATH')
  cls.validate_version(executable)
@@ -187,6 +189,9 @@ def get_version(executable):
  elif platform.system() == 'Darwin':
  # XXX: guess and let the user figure out errors later
  return (0,11,0)
+ elif platform.system() == 'Windows':
+ # XXX: guess and let the user figure out errors later
+ return (0,12,0)
  else:
  print 'WARNING: platform "%s" not supported' % platform.system()
  return None
@@ -197,6 +202,8 @@ def _set_config_dict_value(self, value):
  else:
  if value == '<PATHS>':
  executable = self.find_executable('caffe')
+ if not executable:
+ executable = self.find_executable('caffe.exe')
  else:
  executable = os.path.join(value, 'build', 'tools', 'caffe')
 
@@ -238,7 +245,7 @@ def apply(self):
  print 'Did you forget to "make pycaffe"?'
  raise
 
- if platform.system() == 'Darwin':
+ if platform.system() == 'Darwin' or platform.system() == 'Windows':
  # Strange issue with protocol buffers and pickle - see issue #32
  sys.path.insert(0, os.path.join(
  os.path.dirname(caffe.__file__), 'proto'))

digits/dataset/images/classification/job.py

@@ -37,7 +37,7 @@ def __setstate__(self, state):
  import numpy as np
 
  old_blob = caffe_pb2.BlobProto()
- with open(task.path(task.mean_file)) as infile:
+ with open(task.path(task.mean_file),'rb') as infile:
  old_blob.ParseFromString(infile.read())
  data = np.array(old_blob.data).reshape(
  old_blob.channels,
@@ -48,7 +48,7 @@ def __setstate__(self, state):
  new_blob.num = 1
  new_blob.channels, new_blob.height, new_blob.width = data.shape
  new_blob.data.extend(data.astype(float).flat)
- with open(task.path(task.mean_file), 'w') as outfile:
+ with open(task.path(task.mean_file), 'wb') as outfile:
  outfile.write(new_blob.SerializeToString())
  else:
  print '\tSetting "%s" status to ERROR because it was created with RGB channels' % self.name()

digits/dataset/images/generic/test_lmdb_creator.py

@@ -65,11 +65,11 @@ def create_lmdbs(folder, image_width=None, image_height=None, image_count=None):
  ('train', train_image_count),
  ('val', val_image_count)]:
  image_db = lmdb.open(os.path.join(folder, '%s_images' % phase),
- map_size=1024**4, # 1TB
+ map_size=100000000,
  map_async=True,
  max_dbs=0)
  label_db = lmdb.open(os.path.join(folder, '%s_labels' % phase),
- map_size=1024**4, # 1TB
+ map_size=100000000,
  map_async=True,
  max_dbs=0)
 
@@ -151,7 +151,7 @@ def _save_mean(mean, filename):
  blob.channels = 1
  blob.height, blob.width = mean.shape
  blob.data.extend(mean.astype(float).flat)
- with open(filename, 'w') as outfile:
+ with open(filename, 'wb') as outfile:
  outfile.write(blob.SerializeToString())
 
  elif filename.endswith(('.jpg', '.jpeg', '.png')):

digits/device_query.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python
 # Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
 
 import ctypes
@@ -111,12 +111,25 @@ def get_library(name):
  return ctypes.cdll.LoadLibrary('%s.so' % name)
  elif platform.system() == 'Darwin':
  return ctypes.cdll.LoadLibrary('%s.dylib' % name)
+ elif platform.system() == 'Windows':
+ return ctypes.windll.LoadLibrary('%s.dll' % name)
  except OSError:
  pass
  return None
 
 devices = None
 
+def get_cudart():
+ cudart = get_library('libcudart')
+ if not cudart is None:
+ return cudart
+
+ for ver in range(90,50,-5):
+ for arch in [32, 64]:
+ cudart = get_library('cudart%d_%d' % (arch, ver))
+ if not cudart is None:
+ return cudart
+
 def get_devices(force_reload=False):
  """
  Returns a list of c_cudaDeviceProp's
@@ -131,7 +144,7 @@ def get_devices(force_reload=False):
  return devices
  devices = []
 
- cudart = get_library('libcudart')
+ cudart = get_cudart()
  if cudart is None:
  return []
 
@@ -180,7 +193,9 @@ def get_nvml_info(device_id):
 
  nvml = get_library('libnvidia-ml')
  if nvml is None:
- return None
+ nvml = get_library('nvml')
+ if nvml is None:
+ return None
 
  rc = nvml.nvmlInit()
  if rc != 0:

digits/model/images/classification/test_views.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
 
 import re
 import os
@@ -338,7 +338,7 @@ def test_classify_one(self):
  category = self.imageset_paths.keys()[0]
  image_path = self.imageset_paths[category][0]
  image_path = os.path.join(self.imageset_folder, image_path)
- with open(image_path) as infile:
+ with open(image_path,'rb') as infile:
  # StringIO wrapping is needed to simulate POST file upload.
  image_upload = (StringIO(infile.read()), 'image.png')
 
@@ -360,7 +360,7 @@ def test_classify_one_json(self):
  category = self.imageset_paths.keys()[0]
  image_path = self.imageset_paths[category][0]
  image_path = os.path.join(self.imageset_folder, image_path)
- with open(image_path) as infile:
+ with open(image_path,'rb') as infile:
  # StringIO wrapping is needed to simulate POST file upload.
  image_upload = (StringIO(infile.read()), 'image.png')
 

digits/model/images/classification/views.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
 
 import os
 import re
@@ -246,7 +246,7 @@ def image_classification_model_classify_one():
  if 'image_url' in flask.request.form and flask.request.form['image_url']:
  image = utils.image.load_image(flask.request.form['image_url'])
  elif 'image_file' in flask.request.files and flask.request.files['image_file']:
- with tempfile.NamedTemporaryFile() as outfile:
+ with tempfile.NamedTemporaryFile(suffix='.bin') as outfile:
  flask.request.files['image_file'].save(outfile.name)
  image = utils.image.load_image(outfile.name)
  else:

digits/model/images/generic/test_views.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
 
 import re
 import os
@@ -338,7 +338,7 @@ def test_model_json(self):
 
  def test_infer_one(self):
  image_path = os.path.join(self.imageset_folder, self.test_image)
- with open(image_path) as infile:
+ with open(image_path,'rb') as infile:
  # StringIO wrapping is needed to simulate POST file upload.
  image_upload = (StringIO(infile.read()), 'image.png')
 
@@ -355,7 +355,7 @@ def test_infer_one(self):
 
  def test_infer_one_json(self):
  image_path = os.path.join(self.imageset_folder, self.test_image)
- with open(image_path) as infile:
+ with open(image_path,'rb') as infile:
  # StringIO wrapping is needed to simulate POST file upload.
  image_upload = (StringIO(infile.read()), 'image.png')
 

digits/model/images/generic/views.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
 
 import os
 import re
@@ -222,7 +222,7 @@ def generic_image_model_infer_one():
  if 'image_url' in flask.request.form and flask.request.form['image_url']:
  image = utils.image.load_image(flask.request.form['image_url'])
  elif 'image_file' in flask.request.files and flask.request.files['image_file']:
- with tempfile.NamedTemporaryFile() as outfile:
+ with tempfile.NamedTemporaryFile(suffix='.bin') as outfile:
  flask.request.files['image_file'].save(outfile.name)
  image = utils.image.load_image(outfile.name)
  else:

digits/model/tasks/caffe_train.py

@@ -233,7 +233,7 @@ def save_files_classification(self):
  val_data_layer.data_param.backend = caffe_pb2.DataParameter.LMDB
  if self.use_mean:
  mean_pixel = None
- with open(self.dataset.path(self.dataset.train_db_task().mean_file)) as f:
+ with open(self.dataset.path(self.dataset.train_db_task().mean_file),'rb') as f:
  blob = caffe_pb2.BlobProto()
  blob.MergeFromString(f.read())
  mean = np.reshape(blob.data,
@@ -1312,7 +1312,7 @@ def get_transformer(self):
  channel_swap = (2,1,0)
 
  if self.use_mean:
- with open(self.dataset.path(self.dataset.train_db_task().mean_file)) as infile:
+ with open(self.dataset.path(self.dataset.train_db_task().mean_file),'rb') as infile:
  blob = caffe_pb2.BlobProto()
  blob.MergeFromString(infile.read())
  mean_pixel = np.reshape(blob.data,
@@ -1331,7 +1331,7 @@ def get_transformer(self):
  channel_swap = (2,1,0)
 
  if self.dataset.mean_file:
- with open(self.dataset.path(self.dataset.mean_file)) as infile:
+ with open(self.dataset.path(self.dataset.mean_file),'rb') as infile:
  blob = caffe_pb2.BlobProto()
  blob.MergeFromString(infile.read())
  mean_pixel = np.reshape(blob.data,

digits/task.py

@@ -15,6 +15,8 @@
 from config import config_value
 from status import Status, StatusCls
 
+import platform
+
 # NOTE: Increment this everytime the pickled version changes
 PICKLE_VERSION = 1
 
@@ -128,7 +130,7 @@ def path(self, filename, relative=False):
  path = os.path.join(self.job_dir, filename)
  if relative:
  path = os.path.relpath(path, config_value('jobs_dir'))
- return str(path)
+ return str(path).replace("\\","/")
 
  def ready_to_queue(self):
  """
@@ -193,7 +195,7 @@ def run(self, resources):
  stdout=subprocess.PIPE,
  stderr=subprocess.STDOUT,
  cwd=self.job_dir,
- close_fds=True,
+ close_fds=False if platform.system() == 'Windows' else True,
  )
 
  try:

digits/utils/__init__.py

@@ -2,17 +2,23 @@
 
 import os
 import math
-import fcntl
 import locale
 from random import uniform
 from urlparse import urlparse
 from io import BlockingIOError
 import inspect
+import platform
+
+
+if not platform.system() == 'Windows':
+ import fcntl
+else:
+ import gevent.os
 
 HTTP_TIMEOUT = 6.05
 
 def is_url(url):
- return url is not None and urlparse(url).scheme != ""
+ return url is not None and urlparse(url).scheme != "" and not os.path.exists(url)
 
 def wait_time():
  """Wait a random number of seconds"""
@@ -27,22 +33,25 @@ def nonblocking_readlines(f):
  Newlines are normalized to the Unix standard.
  """
  fd = f.fileno()
- fl = fcntl.fcntl(fd, fcntl.F_GETFL)
- fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+ if not platform.system() == 'Windows':
+ fl = fcntl.fcntl(fd, fcntl.F_GETFL)
+ fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
  enc = locale.getpreferredencoding(False)
 
  buf = bytearray()
  while True:
  try:
- block = os.read(fd, 8192)
+ if not platform.system() == 'Windows':
+ block = os.read(fd, 8192)
+ else:
+ block = gevent.os.tp_read(fd, 8192)
  except (BlockingIOError, OSError):
  yield ""
  continue
 
  if not block:
  if buf:
  yield buf.decode(enc)
- buf.clear()
  break
 
  buf.extend(block)

examples/classification/example.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python
 # Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
 
 """
@@ -65,7 +65,7 @@ def get_transformer(deploy_file, mean_file=None):
 
  if mean_file:
  # set mean pixel
- with open(mean_file) as infile:
+ with open(mean_file,'rb') as infile:
  blob = caffe_pb2.BlobProto()
  blob.MergeFromString(infile.read())
  if blob.HasField('shape'):

scripts/generate_docs.py

@@ -157,10 +157,10 @@ def _print_route(self, route):
  )
  filename = os.path.normpath(route['location']['filename'])
  if filename.startswith(digits_root):
- filename = os.path.relpath(filename, digits_root)
+ filename = os.path.relpath(filename, digits_root).replace("\\","/")
  self.w('Location: [`%s@%s`](%s#L%s)' % (
  filename, route['location']['line'],
- os.path.join('..', filename), route['location']['line'],
+ os.path.join('..', filename).replace("\\","/"), route['location']['line'],
  ))
  self.w()
 

tools/create_db.py

@@ -275,7 +275,7 @@ def create(self, input_file, width, height,
  blob.channels, blob.height, blob.width = data.shape
  blob.data.extend(data.astype(float).flat)
 
- with open(mean_file, 'w') as outfile:
+ with open(mean_file, 'wb') as outfile:
  outfile.write(blob.SerializeToString())
  elif mean_file.lower().endswith(('.jpg', '.jpeg', '.png')):
  image = PIL.Image.fromarray(mean)

tools/test_create_db.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved.
 
 import os.path
 import tempfile
@@ -95,7 +95,7 @@ def setUpClass(cls):
  cls.db_name = tempfile.mkdtemp(dir=cls.tmpdir)
  cls.db = _.DbCreator(cls.db_name)
  _handle, cls.image_path = tempfile.mkstemp(dir=cls.tmpdir, suffix='.jpg')
- with open(cls.image_path, 'w') as outfile:
+ with open(cls.image_path, 'wb') as outfile:
  PIL.Image.fromarray(np.zeros((10,10,3),dtype=np.uint8)).save(outfile, format='JPEG', quality=100)
 
  @classmethod