Merge pull request #549 from Tokutek/backport/7115

backport authentication improvements in upstream v2.4

fixes #527

.gitignore

@@ -98,6 +98,7 @@ scratch
 /libmongoclient.*
 /libmongotestfiles.*
 /libmongoshellfiles.*
+/sharedclient
 
 # plugins
 /plugins/

SConscript.smoke

@@ -61,30 +61,37 @@ def addSmoketest( name, deps, extraSmokeArgs=[] ):
  smokeArgs = smokeFlags + [target] + extraSmokeArgs
  addTest(name, deps, utils.run_smoke_command(*smokeArgs))
 
-def addSmokeSuite( name, suitefile ):
+def addSmokeSuite( name, suitefile, needMongod=False ):
  # Add a smoketest target which invokes smoke.py with
  # --from-file, and passes the named suitefile as the
  # command line argument.
 
  # resolve an initial # in the suitefile
  suitefile = str(env.File(suitefile))
 
- args = smokeFlags + ['--mode', 'files', '--from-file', suitefile]
- addTest(name, [suitefile],
- utils.run_smoke_command(*args))
+ smoke_args = smokeFlags + ['--mode', 'files', '--from-file', suitefile]
+ if not needMongod:
+ smoke_args.append('--dont-start-mongod')
+ addTest(name, [suitefile], utils.run_smoke_command(*smoke_args))
 
 addSmoketest( "smoke", [ add_exe( "test" ), add_exe( "mongod" ), add_exe( "mongo" ) ] )
 addSmoketest( "smokePerf", [ add_exe("perftest") ] )
-addSmoketest( "smokeClient", [
- add_exe('firstExample'),
- add_exe('rsExample'),
- add_exe('secondExample'),
- add_exe('whereExample'),
- add_exe('authTest'),
- add_exe('httpClientTest'),
- add_exe('bsondemo'),
- add_exe('clientTest'),
- ] )
+
+smokeClientDeps = [
+ add_exe('firstExample'),
+ add_exe('rsExample'),
+ add_exe('secondExample'),
+ add_exe('whereExample'),
+ add_exe('authTest'),
+ add_exe('httpClientTest'),
+ add_exe('clientTest'),
+]
+if has_option("sharedclient"):
+ smokeClientDeps += [ "sharedclient/" + name for name in smokeClientDeps ]
+smokeClientDeps += [add_exe('bsondemo')]
+smokeClientDeps += [add_exe('mongod'), add_exe('mongo')]
+smokeClient = addSmoketest( "smokeClient", smokeClientDeps )
+
 addSmoketest( "mongosTest", [ add_exe( 'mongos' ) ])
 addSmokeSuite( "smokeCppUnittests", "$UNITTEST_LIST" )
 addSmokeSuite( "smokeModuleTests", "$MODULETEST_LIST" )

SConstruct

@@ -218,6 +218,8 @@ add_option("smoke-server-opts", "additional options for the mongod for smoke tes
 add_option("smoke-quiet", "make smoke.py be quiet", 0 , False )
 add_option("smokeauth", "run smoke tests with --auth", 0 , False )
 
+add_option("use-sasl-client", "Support SASL authentication in the client library", 0, False)
+
 add_option( "use-system-tcmalloc", "use system version of tcmalloc library", 0, True )
 
 add_option( "use-system-pcre", "use system version of pcre library", 0, True )
@@ -320,7 +322,10 @@ env['_LIBDEPS'] = '$_LIBDEPS_OBJS'
 if has_option('mute'):
  env.Append( CCCOMSTR = "Compiling $TARGET" )
  env.Append( CXXCOMSTR = env["CCCOMSTR"] )
+ env.Append( SHCCCOMSTR = "Compiling $TARGET" )
+ env.Append( SHCXXCOMSTR = env["SHCCCOMSTR"] )
  env.Append( LINKCOMSTR = "Linking $TARGET" )
+ env.Append( SHLINKCOMSTR = env["LINKCOMSTR"] )
  env.Append( ARCOMSTR = "Generating library $TARGET" )
 
 if has_option('mongod-concurrency-level'):
@@ -722,7 +727,7 @@ if nix:
  pass
 
  if linux and has_option( "sharedclient" ):
- env.Append( LINKFLAGS=" -Wl,--as-needed -Wl,-zdefs " )
+ env.Append( SHLINKFLAGS=" -Wl,--as-needed -Wl,-zdefs " )
 
  if linux and has_option( "gcov" ):
  env.Append( CXXFLAGS=" -fprofile-arcs -ftest-coverage " )
@@ -860,6 +865,12 @@ def doConfigure(myenv):
  if not conf.CheckLib( v8_lib_choices ):
  Exit(1)
 
+ conf.env['MONGO_BUILD_SASL_CLIENT'] = bool(has_option("use-sasl-client"))
+ if conf.env['MONGO_BUILD_SASL_CLIENT'] and not conf.CheckLibWithHeader(
+ "sasl2", "sasl/sasl.h", "C", "sasl_version_info(0, 0, 0, 0, 0, 0);", autoadd=False):
+
+ Exit(1)
+
  # requires ports devel/libexecinfo to be installed
  if freebsd or openbsd:
  if not conf.CheckLib("execinfo"):
@@ -1101,13 +1112,6 @@ if len(COMMAND_LINE_TARGETS) > 0 and 'uninstall' in COMMAND_LINE_TARGETS:
  BUILD_TARGETS.remove("uninstall")
  BUILD_TARGETS.append("install")
 
-clientEnv = env.Clone()
-clientEnv['CPPDEFINES'].remove('MONGO_EXPOSE_MACROS')
-
-if not use_system_version_of_library("boost"):
- clientEnv.Append(LIBS=['boost_thread', 'boost_filesystem', 'boost_system'])
- clientEnv.Prepend(LIBPATH=['$BUILD_DIR/third_party/boost/'])
-
 module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules)
 
 # The following symbols are exported for use in subordinate SConscript files.
@@ -1119,7 +1123,6 @@ module_sconscripts = moduleconfig.get_module_sconscripts(mongo_modules)
 # conditional decision making that hasn't been moved up to this SConstruct file,
 # and they are exported here, as well.
 Export("env")
-Export("clientEnv")
 Export("shellEnv")
 Export("testEnv")
 Export("has_option use_system_version_of_library")

buildscripts/smoke.py

@@ -69,6 +69,7 @@
 shell_executable = None
 continue_on_failure = None
 file_of_commands_mode = False
+start_mongod = True
 
 tests = []
 winners = []
@@ -212,6 +213,8 @@ def start(self):
  """
  utils.ensureDir(dir_name)
  argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
+ # This should always be set for tests
+ argv += ['--setParameter', 'enableTestCommands=1']
  if self.kwargs.get('small_oplog'):
  argv += ["--master", "--oplogSize", "511"]
  if self.kwargs.get('small_oplog_rs'):
@@ -224,11 +227,16 @@ def start(self):
  argv += ['--nopreallocj']
  if self.kwargs.get('auth'):
  argv += ['--auth']
+ authMechanism = self.kwargs.get('authMechanism', 'MONGODB-CR')
+ if authMechanism != 'MONGODB-CR':
+ argv.append('--setParameter=authenticationMechanisms=' + authMechanism)
  self.auth = True
  if len(server_log_file) > 0:
  argv += ['--logpath', server_log_file]
  if len(smoke_server_opts) > 0:
  argv += [smoke_server_opts]
+ if self.kwargs.get('use_ssl'):
+ argv += ['--sslOnNormalPorts', '--sslPEMKeyFile', 'jstests/libs/smoke.pem']
  if not quiet:
  print "running " + " ".join(argv)
  self.proc = self._start(buildlogger(argv, is_global=True))
@@ -383,6 +391,23 @@ def skipTest(path):
  # These tests fail due to bugs
  if os.path.join(parentDir,basename) in ["sharding/sync_conn_cmd.js"]:
  return True
+
+ authTestsToSkip = [("sharding", "gle_with_conf_servers.js"), # SERVER-6972
+ ("sharding", "read_pref.js"), # SERVER-6972
+ ("sharding", "read_pref_cmd.js"), # SERVER-6972
+ ("sharding", "read_pref_rs_client.js"), # SERVER-6972
+ ("sharding", "sync_conn_cmd.js"), #SERVER-6327
+ ("sharding", "sync3.js"), # SERVER-6388 for this and those below
+ ("sharding", "sync6.js"),
+ ("sharding", "parallel.js"),
+ ("jstests", "bench_test1.js"),
+ ("jstests", "bench_test2.js"),
+ ("jstests", "bench_test3.js"),
+ ]
+
+ if os.path.join(parentDir,basename) in [ os.path.join(*test) for test in authTestsToSkip ]:
+ return True
+
  if _debug:
  # MutexDebugger complains about this test, it's not unique to tokumx though, see #79
  if os.path.join(parentDir,basename) in ["sharding/remove2.js"]:
@@ -415,11 +440,13 @@ def runTest(test, testnum):
  if os.path.basename(path) in ('python', 'python.exe'):
  path = argv[1]
  elif ext == ".js":
- argv = [shell_executable, "--port", mongod_port]
+ argv = [shell_executable, "--port", mongod_port, '--authenticationMechanism', authMechanism]
  if not usedb:
  argv += ["--nodb"]
  if small_oplog or small_oplog_rs:
  argv += ["--eval", 'testingReplication = true;']
+ if use_ssl:
+ argv += ["--ssl"]
  argv += [path]
  elif ext in ["", ".exe"]:
  # Blech.
@@ -526,12 +553,14 @@ def runTest(test, testnum):
  finally:
  tempfile.close()
 
-"""
- try:
- c = Connection( "127.0.0.1" , int(mongod_port) )
- except Exception,e:
- raise TestServerFailure(path)
-"""
+ if r != 0:
+ raise TestExitFailure(path, r)
+
+ if start_mongod:
+ try:
+ c = Connection(host="127.0.0.1", port=int(mongod_port), ssl=use_ssl)
+ except Exception,e:
+ raise TestServerFailure(path)
 
 def run_tests(tests):
  # FIXME: some suites of tests start their own mongod, so don't
@@ -542,12 +571,28 @@ def run_tests(tests):
  # The reason we want to use "with" is so that we get __exit__ semantics
  # but "with" is only supported on Python 2.5+
 
- master = mongod(small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__()
+ if start_mongod:
+ master = mongod(small_oplog_rs=small_oplog_rs,
+ small_oplog=small_oplog,
+ no_journal=no_journal,
+ no_preallocj=no_preallocj,
+ auth=auth,
+ authMechanism=authMechanism,
+ use_ssl=use_ssl).__enter__()
+ else:
+ master = Nothing()
  try:
  if small_oplog:
  slave = mongod(slave=True).__enter__()
  elif small_oplog_rs:
- slave = mongod(slave=True,small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__()
+ slave = mongod(slave=True,
+ small_oplog_rs=small_oplog_rs,
+ small_oplog=small_oplog,
+ no_journal=no_journal,
+ no_preallocj=no_preallocj,
+ auth=auth,
+ authMechanism=authMechanism,
+ use_ssl=use_ssl).__enter__()
  primary = Connection(port=master.port, slave_okay=True);
 
  primary.admin.command({'replSetInitiate' : {'_id' : 'foo', 'members' : [
@@ -582,7 +627,13 @@ def run_tests(tests):
  if (tests_run+1) % 20 == 0:
  # restart mongo every 20 times, for our 32-bit machines
  master.__exit__(None, None, None)
- master = mongod(small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__()
+ master = mongod(small_oplog_rs=small_oplog_rs,
+ small_oplog=small_oplog,
+ no_journal=no_journal,
+ no_preallocj=no_preallocj,
+ auth=auth,
+ authMechanism=authMechanism,
+ use_ssl=use_ssl).__enter__()
 
  except TestFailure, f:
  try:
@@ -676,6 +727,12 @@ def expand_suites(suites,expandUseDB=True):
  paths = ["firstExample", "secondExample", "whereExample", "authTest", "clientTest", "httpClientTest"]
  if os.sys.platform == "win32":
  paths = [path + '.exe' for path in paths]
+
+ # Add any files of the same name from the sharedclient directory.
+ scpaths = ["sharedclient/" + path for path in paths]
+ scfiles = glob.glob("sharedclient/*")
+ paths += [scfile for scfile in scfiles if scfile in scpaths]
+
  # hack
  tests += [(test_path and path or os.path.join(mongo_repo, path), False) for path in paths]
  elif suite == 'mongosTest':
@@ -719,8 +776,12 @@ def add_exe(e):
  return e
 
 def set_globals(options, tests):
- global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, small_oplog_rs, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, smoke_server_opts, server_log_file, tests_log, quiet, test_path
+ global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, small_oplog_rs
+ global no_journal, no_preallocj, auth, authMechanism, keyFile, smoke_db_prefix, smoke_server_opts, server_log_file, tests_log, quiet, test_path, start_mongod
+ global use_ssl
  global file_of_commands_mode
+ start_mongod = options.start_mongod
+ use_ssl = options.use_ssl
  #Careful, this can be called multiple times
  test_path = options.test_path
 
@@ -748,8 +809,10 @@ def set_globals(options, tests):
  print "Not running client suite with auth even though --auth was provided"
  auth = False;
  keyFile = False;
+ authMechanism = None
  else:
  auth = options.auth
+ authMechanism = options.authMechanism
  keyFile = options.keyFile
 
  if auth and not keyFile:
@@ -882,6 +945,8 @@ def main():
  parser.add_option('--auth', dest='auth', default=False,
  action="store_true",
  help='Run standalone mongods in tests with authentication enabled')
+ parser.add_option('--authMechanism', dest='authMechanism', default='MONGODB-CR',
+ help='Use the given authentication mechanism, when --auth is used.')
  parser.add_option('--keyFile', dest='keyFile', default=None,
  help='Path to keyFile to use to run replSet and sharding tests with authentication enabled')
  parser.add_option('--ignore', dest='ignore_files', default=None,
@@ -908,6 +973,12 @@ def main():
  parser.add_option('--skip-until', dest='skip_tests_until', default="",
  action="store",
  help='Skip all tests alphabetically less than the given name.')
+ parser.add_option('--dont-start-mongod', dest='start_mongod', default=True, 
+ action='store_false',
+ help='Do not start mongod before commencing test running')
+ parser.add_option('--use-ssl', dest='use_ssl', default=False,
+ action='store_true',
+ help='Run mongo shell and mongod instances with SSL encryption')
 
  # Buildlogger invocation from command line
  parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,