diff --git a/lib/file-list.js b/lib/file-list.js
index 13129cd32..887a9d71e 100644
--- a/lib/file-list.js
+++ b/lib/file-list.js
@@ -1,8 +1,5 @@
 'use strict'
 
-// Dependencies
-// ------------
-
 const Promise = require('bluebird')
 const mm = require('minimatch')
 const Glob = require('glob').Glob
@@ -16,127 +13,56 @@ const helper = require('./helper')
 const log = require('./logger').create('watcher')
 const createPatternObject = require('./config').createPatternObject
 
-// Constants
-// ---------
-
-const GLOB_OPTS = {
-  cwd: '/',
-  follow: true,
-  nodir: true,
-  sync: true
-}
-
-// Helper Functions
-// ----------------
-
-const byPath = (a, b) => {
+function byPath (a, b) {
   if (a.path > b.path) return 1
   if (a.path < b.path) return -1
 
   return 0
 }
 
-/**
- * The List is an object for tracking all files that karma knows about
- * currently.
- */
 class FileList {
-  /**
-   * @param {Array} patterns
-   * @param {Array} excludes
-   * @param {EventEmitter} emitter
-   * @param {Function} preprocess
-   * @param {number} autoWatchBatchDelay
-   */
   constructor (patterns, excludes, emitter, preprocess, autoWatchBatchDelay) {
-    // Store options
-    this._patterns = patterns
-    this._excludes = excludes
+    this._patterns = patterns || []
+    this._excludes = excludes || []
     this._emitter = emitter
     this._preprocess = Promise.promisify(preprocess)
-    this._autoWatchBatchDelay = autoWatchBatchDelay
 
-    // The actual list of files
     this.buckets = new Map()
 
-    // Internal tracker if we are refreshing.
-    // When a refresh is triggered this gets set
-    // to the promise that `this._refresh` returns.
-    // So we know we are refreshing when this promise
-    // is still pending, and we are done when it's either
-    // resolved or rejected.
     this._refreshing = Promise.resolve()
 
-    // Emit the `file_list_modified` event.
-    // This function is debounced to the value of `autoWatchBatchDelay`
-    // to avoid reloading while files are still being modified.
     const emit = () => {
       this._emitter.emit('file_list_modified', this.files)
     }
 
-    const debouncedEmit = _.debounce(emit, this._autoWatchBatchDelay)
+    const debouncedEmit = _.debounce(emit, autoWatchBatchDelay)
     this._emitModified = (immediate) => {
       immediate ? emit() : debouncedEmit()
     }
   }
 
-  // Private Interface
-  // -----------------
-
-  // Is the given path matched by any exclusion filter
-  //
-  // path - String
-  //
-  // Returns `undefined` if no match, otherwise the matching
-  // pattern.
-  _isExcluded (path) {
-    return _.find(this._excludes, (pattern) => mm(path, pattern))
+  _findExcluded (path) {
+    return this._excludes.find((pattern) => mm(path, pattern))
   }
 
-  // Find the matching include pattern for the given path.
-  //
-  // path - String
-  //
-  // Returns the match or `undefined` if none found.
-  _isIncluded (path) {
-    return _.find(this._patterns, (pattern) => mm(path, pattern.pattern))
+  _findIncluded (path) {
+    return this._patterns.find((pattern) => mm(path, pattern.pattern))
   }
 
-  // Find the given path in the bucket corresponding
-  // to the given pattern.
-  //
-  // path    - String
-  // pattern - Object
-  //
-  // Returns a File or undefined
   _findFile (path, pattern) {
     if (!path || !pattern) return
-    if (!this.buckets.has(pattern.pattern)) return
-
-    return _.find(Array.from(this.buckets.get(pattern.pattern)), (file) => {
-      return file.originalPath === path
-    })
+    return this._getFilesByPattern(pattern.pattern).find((file) => file.originalPath === path)
   }
 
-  // Is the given path already in the files list.
-  //
-  // path - String
-  //
-  // Returns a boolean.
   _exists (path) {
-    const patterns = this._patterns.filter((pattern) => mm(path, pattern.pattern))
-
-    return !!_.find(patterns, (pattern) => this._findFile(path, pattern))
+    return !!this._patterns.find((pattern) => mm(path, pattern.pattern) && this._findFile(path, pattern))
   }
 
-  // Check if we are currently refreshing
-  _isRefreshing () {
-    return this._refreshing.isPending()
+  _getFilesByPattern (pattern) {
+    return this.buckets.get(pattern) || []
   }
 
-  // Do the actual work of refreshing
   _refresh () {
-    const buckets = this.buckets
     const matchedFiles = new Set()
 
     let promise
@@ -145,21 +71,20 @@ class FileList {
       const type = patternObject.type
 
       if (helper.isUrlAbsolute(pattern)) {
-        buckets.set(pattern, new Set([new Url(pattern, type)]))
+        this.buckets.set(pattern, [new Url(pattern, type)])
         return Promise.resolve()
       }
 
-      const mg = new Glob(pathLib.normalize(pattern), GLOB_OPTS)
+      const mg = new Glob(pathLib.normalize(pattern), { cwd: '/', follow: true, nodir: true, sync: true })
       const files = mg.found
-      buckets.set(pattern, new Set())
-
       if (_.isEmpty(files)) {
+        this.buckets.set(pattern, [])
         log.warn('Pattern "%s" does not match any file.', pattern)
         return
       }
 
       return Promise.map(files, (path) => {
-        if (this._isExcluded(path)) {
+        if (this._findExcluded(path)) {
           log.debug('Excluded file "%s"', path)
           return Promise.resolve()
         }
@@ -170,27 +95,20 @@ class FileList {
 
         matchedFiles.add(path)
 
-        const mtime = mg.statCache[path].mtime
-        const doNotCache = patternObject.nocache
-        const type = patternObject.type
-        const file = new File(path, mtime, doNotCache, type)
-
+        const file = new File(path, mg.statCache[path].mtime, patternObject.nocache, type)
         if (file.doNotCache) {
           log.debug('Not preprocessing "%s" due to nocache', pattern)
           return Promise.resolve(file)
         }
 
-        return this._preprocess(file).then(() => {
-          return file
-        })
+        return this._preprocess(file).then(() => file)
       })
         .then((files) => {
           files = _.compact(files)
+          this.buckets.set(pattern, files)
 
           if (_.isEmpty(files)) {
             log.warn('All files matched by "%s" were excluded or matched by prior matchers.', pattern)
-          } else {
-            buckets.set(pattern, new Set(files))
           }
         })
     })
@@ -198,7 +116,6 @@ class FileList {
         if (this._refreshing !== promise) {
           return this._refreshing
         }
-        this.buckets = buckets
         this._emitModified(true)
         return this.files
       })
@@ -206,25 +123,10 @@ class FileList {
     return promise
   }
 
-  // Public Interface
-  // ----------------
-
   get files () {
-    const uniqueFlat = (list) => {
-      return _.uniq(_.flatten(list), 'path')
-    }
-
-    const expandPattern = (p) => {
-      return Array.from(this.buckets.get(p.pattern) || []).sort(byPath)
-    }
-
-    const served = this._patterns.filter((pattern) => {
-      return pattern.served
-    })
-      .map(expandPattern)
-
-    const lookup = {}
+    const served = []
     const included = {}
+    const lookup = {}
     this._patterns.forEach((p) => {
       // This needs to be here sadly, as plugins are modifiying
       // the _patterns directly resulting in elements not being
@@ -233,10 +135,16 @@ class FileList {
         p = createPatternObject(p)
       }
 
-      const bucket = expandPattern(p)
-      bucket.forEach((file) => {
+      const files = this._getFilesByPattern(p.pattern)
+      files.sort(byPath)
+      if (p.served) {
+        served.push.apply(served, files) // TODO: replace with served.push(...files) after remove Node 4 support
+      }
+
+      files.forEach((file) => {
         const other = lookup[file.path]
         if (other && other.compare(p) < 0) return
+
         lookup[file.path] = p
         if (p.included) {
           included[file.path] = file
@@ -247,55 +155,31 @@ class FileList {
     })
 
     return {
-      served: uniqueFlat(served),
+      served: _.uniq(served, 'path'),
       included: _.values(included)
     }
   }
 
-  // Reglob all patterns to update the list.
-  //
-  // Returns a promise that is resolved when the refresh
-  // is completed.
   refresh () {
     this._refreshing = this._refresh()
     return this._refreshing
   }
 
-  // Set new patterns and excludes and update
-  // the list accordingly
-  //
-  // patterns - Array, the new patterns.
-  // excludes - Array, the new exclude patterns.
-  //
-  // Returns a promise that is resolved when the refresh
-  // is completed.
   reload (patterns, excludes) {
-    this._patterns = patterns
-    this._excludes = excludes
+    this._patterns = patterns || []
+    this._excludes = excludes || []
 
-    // Wait until the current refresh is done and then do a
-    // refresh to ensure a refresh actually happens
     return this.refresh()
   }
 
-  // Add a new file from the list.
-  // This is called by the watcher
-  //
-  // path - String, the path of the file to update.
-  //
-  // Returns a promise that is resolved when the update
-  // is completed.
   addFile (path) {
-    // Ensure we are not adding a file that should be excluded
-    const excluded = this._isExcluded(path)
+    const excluded = this._findExcluded(path)
     if (excluded) {
       log.debug('Add file "%s" ignored. Excluded by "%s".', path, excluded)
-
       return Promise.resolve(this.files)
     }
 
-    const pattern = this._isIncluded(path)
-
+    const pattern = this._findIncluded(path)
     if (!pattern) {
       log.debug('Add file "%s" ignored. Does not match any pattern.', path)
       return Promise.resolve(this.files)
@@ -307,15 +191,16 @@ class FileList {
     }
 
     const file = new File(path)
-    this.buckets.get(pattern.pattern).add(file)
+    this._getFilesByPattern(pattern.pattern).push(file)
 
     return Promise.all([
       fs.statAsync(path),
       this._refreshing
-    ]).spread((stat) => {
-      file.mtime = stat.mtime
-      return this._preprocess(file)
-    })
+    ])
+      .spread((stat) => {
+        file.mtime = stat.mtime
+        return this._preprocess(file)
+      })
       .then(() => {
         log.info('Added file "%s".', path)
         this._emitModified()
@@ -323,18 +208,11 @@ class FileList {
       })
   }
 
-  // Update the `mtime` of a file.
-  // This is called by the watcher
-  //
-  // path - String, the path of the file to update.
-  //
-  // Returns a promise that is resolved when the update
-  // is completed.
   changeFile (path) {
-    const pattern = this._isIncluded(path)
+    const pattern = this._findIncluded(path)
     const file = this._findFile(path, pattern)
 
-    if (!pattern || !file) {
+    if (!file) {
       log.debug('Changed file "%s" ignored. Does not match any file in the list.', path)
       return Promise.resolve(this.files)
     }
@@ -353,42 +231,31 @@ class FileList {
         this._emitModified()
         return this.files
       })
-      .catch(Promise.CancellationError, () => {
-        return this.files
-      })
+      .catch(Promise.CancellationError, () => this.files)
   }
 
-  // Remove a file from the list.
-  // This is called by the watcher
-  //
-  // path - String, the path of the file to update.
-  //
-  // Returns a promise that is resolved when the update
-  // is completed.
   removeFile (path) {
     return Promise.try(() => {
-      const pattern = this._isIncluded(path)
+      const pattern = this._findIncluded(path)
       const file = this._findFile(path, pattern)
 
-      if (!pattern || !file) {
+      if (file) {
+        helper.arrayRemove(this._getFilesByPattern(pattern.pattern), file)
+        log.info('Removed file "%s".', path)
+
+        this._emitModified()
+      } else {
         log.debug('Removed file "%s" ignored. Does not match any file in the list.', path)
-        return this.files
       }
-
-      this.buckets.get(pattern.pattern).delete(file)
-
-      log.info('Removed file "%s".', path)
-      this._emitModified()
       return this.files
     })
   }
 }
 
-FileList.factory = function (patterns, excludes, emitter, preprocess, autoWatchBatchDelay) {
-  return new FileList(patterns, excludes, emitter, preprocess, autoWatchBatchDelay)
+FileList.factory = function (config, emitter, preprocess) {
+  return new FileList(config.files, config.exclude, emitter, preprocess, config.autoWatchBatchDelay)
 }
 
-FileList.factory.$inject = ['config.files', 'config.exclude', 'emitter', 'preprocess',
-  'config.autoWatchBatchDelay']
+FileList.factory.$inject = ['config', 'emitter', 'preprocess']
 
 module.exports = FileList
diff --git a/test/unit/file-list.spec.js b/test/unit/file-list.spec.js
index 42a61a6e1..b4a99d3b6 100644
--- a/test/unit/file-list.spec.js
+++ b/test/unit/file-list.spec.js
@@ -11,15 +11,15 @@ const helper = require('../../lib/helper')
 const config = require('../../lib/config')
 
 // create an array of pattern objects from given strings
-const patterns = function () {
+function patterns () {
   return Array.from(arguments).map((str) => new config.Pattern(str))
 }
 
-const pathsFrom = (files) => {
-  return _.map(Array.from(files), 'path')
+function pathsFrom (files) {
+  return Array.from(files).map((file) => file.path)
 }
 
-const findFile = (path, files) => {
+function findFile (path, files) {
   return Array.from(files).find((file) => file.path === path)
 }
 
@@ -159,7 +159,7 @@ describe('FileList', () => {
     })
   })
 
-  describe('_isExcluded', () => {
+  describe('_findExcluded', () => {
     beforeEach(() => {
       preprocess = sinon.spy((file, done) => process.nextTick(done))
       emitter = new EventEmitter()
@@ -167,18 +167,18 @@ describe('FileList', () => {
 
     it('returns undefined when no match is found', () => {
       list = new List([], ['hello.js', 'world.js'], emitter, preprocess)
-      expect(list._isExcluded('hello.txt')).to.be.undefined
-      expect(list._isExcluded('/hello/world/i.js')).to.be.undefined
+      expect(list._findExcluded('hello.txt')).to.be.undefined
+      expect(list._findExcluded('/hello/world/i.js')).to.be.undefined
     })
 
     it('returns the first match if it finds one', () => {
       list = new List([], ['*.js', '**/*.js'], emitter, preprocess)
-      expect(list._isExcluded('world.js')).to.be.eql('*.js')
-      expect(list._isExcluded('/hello/world/i.js')).to.be.eql('**/*.js')
+      expect(list._findExcluded('world.js')).to.be.eql('*.js')
+      expect(list._findExcluded('/hello/world/i.js')).to.be.eql('**/*.js')
     })
   })
 
-  describe('_isIncluded', () => {
+  describe('_findIncluded', () => {
     beforeEach(() => {
       preprocess = sinon.spy((file, done) => process.nextTick(done))
       emitter = new EventEmitter()
@@ -186,14 +186,14 @@ describe('FileList', () => {
 
     it('returns undefined when no match is found', () => {
       list = new List(patterns('*.js'), [], emitter, preprocess)
-      expect(list._isIncluded('hello.txt')).to.be.undefined
-      expect(list._isIncluded('/hello/world/i.js')).to.be.undefined
+      expect(list._findIncluded('hello.txt')).to.be.undefined
+      expect(list._findIncluded('/hello/world/i.js')).to.be.undefined
     })
 
     it('returns the first match if it finds one', () => {
       list = new List(patterns('*.js', '**/*.js'), [], emitter, preprocess)
-      expect(list._isIncluded('world.js').pattern).to.be.eql('*.js')
-      expect(list._isIncluded('/hello/world/i.js').pattern).to.be.eql('**/*.js')
+      expect(list._findIncluded('world.js').pattern).to.be.eql('*.js')
+      expect(list._findIncluded('/hello/world/i.js').pattern).to.be.eql('**/*.js')
     })
   })
 
@@ -288,7 +288,7 @@ describe('FileList', () => {
 
     it('cancels refreshs', () => {
       const checkResult = (files) => {
-        expect(_.map(files.served, 'path')).to.contain('/some/a.js', '/some/b.js', '/some/c.js')
+        expect(pathsFrom(files.served)).to.contain('/some/a.js', '/some/b.js', '/some/c.js')
       }
 
       const p1 = list.refresh().then(checkResult)