diff --git a/pathtools/glob.go b/pathtools/glob.go
index a842a82..481b286 100644
--- a/pathtools/glob.go
+++ b/pathtools/glob.go
@@ -28,25 +28,25 @@
 var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **")
 var GlobLastRecursiveErr = errors.New("pattern ** as last path element")
 
-// Glob returns the list of files that match the given pattern but
-// do not match the given exclude patterns, along with the list of directories
-// that were searched to construct the file list.  The supported glob and
-// exclude patterns are equivalent to filepath.Glob, with an extension that
-// recursive glob (** matching zero or more complete path entries) is supported.
-// Glob also returns a list of directories that were searched.
+// Glob returns the list of files that match the given pattern but do not match
+// the given exclude patterns, along with the list of directories and other
+// dependencies that were searched to construct the file list.  The supported
+// glob and exclude patterns are equivalent to filepath.Glob, with an extension
+// that recursive glob (** matching zero or more complete path entries) is
+// supported.  Glob also returns a list of directories that were searched.
 //
 // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
 // should be used instead, as they will automatically set up dependencies
 // to rerun the primary builder when the list of matching files changes.
-func Glob(pattern string, excludes []string) (matches, dirs []string, err error) {
+func Glob(pattern string, excludes []string) (matches, deps []string, err error) {
 	return startGlob(OsFs, pattern, excludes)
 }
 
-func startGlob(fs FileSystem, pattern string, excludes []string) (matches, dirs []string, err error) {
+func startGlob(fs FileSystem, pattern string, excludes []string) (matches, deps []string, err error) {
 	if filepath.Base(pattern) == "**" {
 		return nil, nil, GlobLastRecursiveErr
 	} else {
-		matches, dirs, err = glob(fs, pattern, false)
+		matches, deps, err = glob(fs, pattern, false)
 	}
 
 	if err != nil {
@@ -58,7 +58,20 @@
 		return nil, nil, err
 	}
 
-	return matches, dirs, nil
+	// If the pattern has wildcards, we added dependencies on the
+	// containing directories to know about changes.
+	//
+	// If the pattern didn't have wildcards, and didn't find matches, the
+	// most specific found directories were added.
+	//
+	// But if it didn't have wildcards, and did find a match, no
+	// dependencies were added, so add the match itself to detect when it
+	// is removed.
+	if !isWild(pattern) {
+		deps = append(deps, matches...)
+	}
+
+	return matches, deps, nil
 }
 
 // glob is a recursive helper function to handle globbing each level of the pattern individually,
@@ -346,7 +359,7 @@
 // should be used instead, as they will automatically set up dependencies
 // to rerun the primary builder when the list of matching files changes.
 func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
-	files, dirs, err := Glob(glob, excludes)
+	files, deps, err := Glob(glob, excludes)
 	if err != nil {
 		return nil, err
 	}
@@ -354,7 +367,7 @@
 	fileList := strings.Join(files, "\n") + "\n"
 
 	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
-	deptools.WriteDepFile(depFile, fileListFile, dirs)
+	deptools.WriteDepFile(depFile, fileListFile, deps)
 
 	return
 }
diff --git a/pathtools/glob_test.go b/pathtools/glob_test.go
index 6e236fc..513b597 100644
--- a/pathtools/glob_test.go
+++ b/pathtools/glob_test.go
@@ -27,68 +27,68 @@
 	pattern  string
 	matches  []string
 	excludes []string
-	dirs     []string
+	deps     []string
 	err      error
 }{
 	// Current directory tests
 	{
 		pattern: "*",
 		matches: []string{"a", "b", "c", "d.ext", "e.ext"},
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 	{
 		pattern: "*.ext",
 		matches: []string{"d.ext", "e.ext"},
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 	{
 		pattern: "*/a",
 		matches: []string{"a/a", "b/a"},
-		dirs:    []string{".", "a", "b", "c"},
+		deps:    []string{".", "a", "b", "c"},
 	},
 	{
 		pattern: "*/*/a",
 		matches: []string{"a/a/a"},
-		dirs:    []string{".", "a", "b", "c", "a/a", "a/b", "c/f", "c/g", "c/h"},
+		deps:    []string{".", "a", "b", "c", "a/a", "a/b", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern: "*/a/a",
 		matches: []string{"a/a/a"},
-		dirs:    []string{".", "a", "b", "c", "a/a"},
+		deps:    []string{".", "a", "b", "c", "a/a"},
 	},
 
 	// ./ directory tests
 	{
 		pattern: "./*",
 		matches: []string{"a", "b", "c", "d.ext", "e.ext"},
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 	{
 		pattern: "./*.ext",
 		matches: []string{"d.ext", "e.ext"},
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 	{
 		pattern: "./*/a",
 		matches: []string{"a/a", "b/a"},
-		dirs:    []string{".", "a", "b", "c"},
+		deps:    []string{".", "a", "b", "c"},
 	},
 	{
 		pattern: "./[ac]/a",
 		matches: []string{"a/a"},
-		dirs:    []string{".", "a", "c"},
+		deps:    []string{".", "a", "c"},
 	},
 
 	// subdirectory tests
 	{
 		pattern: "c/*/*.ext",
 		matches: []string{"c/f/f.ext", "c/g/g.ext"},
-		dirs:    []string{"c", "c/f", "c/g", "c/h"},
+		deps:    []string{"c", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern: "a/*/a",
 		matches: []string{"a/a/a"},
-		dirs:    []string{"a", "a/a", "a/b"},
+		deps:    []string{"a", "a/a", "a/b"},
 	},
 
 	// absolute tests
@@ -98,7 +98,7 @@
 			filepath.Join(pwd, "testdata/c/f/f.ext"),
 			filepath.Join(pwd, "testdata/c/g/g.ext"),
 		},
-		dirs: []string{
+		deps: []string{
 			filepath.Join(pwd, "testdata/c"),
 			filepath.Join(pwd, "testdata/c/f"),
 			filepath.Join(pwd, "testdata/c/g"),
@@ -110,41 +110,41 @@
 	{
 		pattern: "a",
 		matches: []string{"a"},
-		dirs:    nil,
+		deps:    []string{"a"},
 	},
 	{
 		pattern: "a/a",
 		matches: []string{"a/a"},
-		dirs:    nil,
+		deps:    []string{"a/a"},
 	},
 
 	// clean tests
 	{
 		pattern: "./c/*/*.ext",
 		matches: []string{"c/f/f.ext", "c/g/g.ext"},
-		dirs:    []string{"c", "c/f", "c/g", "c/h"},
+		deps:    []string{"c", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern: "c/../c/*/*.ext",
 		matches: []string{"c/f/f.ext", "c/g/g.ext"},
-		dirs:    []string{"c", "c/f", "c/g", "c/h"},
+		deps:    []string{"c", "c/f", "c/g", "c/h"},
 	},
 
 	// recursive tests
 	{
 		pattern: "**/a",
 		matches: []string{"a", "a/a", "a/a/a", "b/a"},
-		dirs:    []string{".", "a", "a/a", "a/b", "b", "c", "c/f", "c/g", "c/h"},
+		deps:    []string{".", "a", "a/a", "a/b", "b", "c", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern: "a/**/a",
 		matches: []string{"a/a", "a/a/a"},
-		dirs:    []string{"a", "a/a", "a/b"},
+		deps:    []string{"a", "a/a", "a/b"},
 	},
 	{
 		pattern: "a/**/*",
 		matches: []string{"a/a", "a/b", "a/a/a", "a/b/b"},
-		dirs:    []string{"a", "a/a", "a/b"},
+		deps:    []string{"a", "a/a", "a/b"},
 	},
 
 	// absolute recursive tests
@@ -156,7 +156,7 @@
 			filepath.Join(pwd, "testdata/c/f/f.ext"),
 			filepath.Join(pwd, "testdata/c/g/g.ext"),
 		},
-		dirs: []string{
+		deps: []string{
 			filepath.Join(pwd, "testdata"),
 			filepath.Join(pwd, "testdata/a"),
 			filepath.Join(pwd, "testdata/a/a"),
@@ -200,31 +200,31 @@
 		pattern:  "*.ext",
 		excludes: []string{"d.ext"},
 		matches:  []string{"e.ext"},
-		dirs:     []string{"."},
+		deps:     []string{"."},
 	},
 	{
 		pattern:  "*/*",
 		excludes: []string{"a/b"},
 		matches:  []string{"a/a", "b/a", "c/c", "c/f", "c/g", "c/h"},
-		dirs:     []string{".", "a", "b", "c"},
+		deps:     []string{".", "a", "b", "c"},
 	},
 	{
 		pattern:  "*/*",
 		excludes: []string{"a/b", "c/c"},
 		matches:  []string{"a/a", "b/a", "c/f", "c/g", "c/h"},
-		dirs:     []string{".", "a", "b", "c"},
+		deps:     []string{".", "a", "b", "c"},
 	},
 	{
 		pattern:  "*/*",
 		excludes: []string{"c/*", "*/a"},
 		matches:  []string{"a/b"},
-		dirs:     []string{".", "a", "b", "c"},
+		deps:     []string{".", "a", "b", "c"},
 	},
 	{
 		pattern:  "*/*",
 		excludes: []string{"*/*"},
 		matches:  nil,
-		dirs:     []string{".", "a", "b", "c"},
+		deps:     []string{".", "a", "b", "c"},
 	},
 
 	// absolute exclude tests
@@ -234,7 +234,7 @@
 		matches: []string{
 			filepath.Join(pwd, "testdata/c/g/g.ext"),
 		},
-		dirs: []string{
+		deps: []string{
 			filepath.Join(pwd, "testdata/c"),
 			filepath.Join(pwd, "testdata/c/f"),
 			filepath.Join(pwd, "testdata/c/g"),
@@ -247,7 +247,7 @@
 		matches: []string{
 			filepath.Join(pwd, "testdata/c/g/g.ext"),
 		},
-		dirs: []string{
+		deps: []string{
 			filepath.Join(pwd, "testdata/c"),
 			filepath.Join(pwd, "testdata/c/f"),
 			filepath.Join(pwd, "testdata/c/g"),
@@ -260,43 +260,43 @@
 		pattern:  "*.ext",
 		excludes: []string{"**/*.ext"},
 		matches:  nil,
-		dirs:     []string{"."},
+		deps:     []string{"."},
 	},
 	{
 		pattern:  "*/*",
 		excludes: []string{"**/b"},
 		matches:  []string{"a/a", "b/a", "c/c", "c/f", "c/g", "c/h"},
-		dirs:     []string{".", "a", "b", "c"},
+		deps:     []string{".", "a", "b", "c"},
 	},
 	{
 		pattern:  "*/*",
 		excludes: []string{"a/**/*"},
 		matches:  []string{"b/a", "c/c", "c/f", "c/g", "c/h"},
-		dirs:     []string{".", "a", "b", "c"},
+		deps:     []string{".", "a", "b", "c"},
 	},
 	{
 		pattern:  "**/*",
 		excludes: []string{"**/*"},
 		matches:  nil,
-		dirs:     []string{".", "a", "a/a", "a/b", "b", "c", "c/f", "c/g", "c/h"},
+		deps:     []string{".", "a", "a/a", "a/b", "b", "c", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern:  "*/*/*",
 		excludes: []string{"a/**/a"},
 		matches:  []string{"a/b/b", "c/f/f.ext", "c/g/g.ext", "c/h/h"},
-		dirs:     []string{".", "a", "b", "c", "a/a", "a/b", "c/f", "c/g", "c/h"},
+		deps:     []string{".", "a", "b", "c", "a/a", "a/b", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern:  "*/*/*",
 		excludes: []string{"**/a"},
 		matches:  []string{"a/b/b", "c/f/f.ext", "c/g/g.ext", "c/h/h"},
-		dirs:     []string{".", "a", "b", "c", "a/a", "a/b", "c/f", "c/g", "c/h"},
+		deps:     []string{".", "a", "b", "c", "a/a", "a/b", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern:  "c/*/*.ext",
 		excludes: []string{"c/**/f.ext"},
 		matches:  []string{"c/g/g.ext"},
-		dirs:     []string{"c", "c/f", "c/g", "c/h"},
+		deps:     []string{"c", "c/f", "c/g", "c/h"},
 	},
 
 	// absoulte recursive exclude tests
@@ -306,7 +306,7 @@
 		matches: []string{
 			filepath.Join(pwd, "testdata/c/g/g.ext"),
 		},
-		dirs: []string{
+		deps: []string{
 			filepath.Join(pwd, "testdata/c"),
 			filepath.Join(pwd, "testdata/c/f"),
 			filepath.Join(pwd, "testdata/c/g"),
@@ -319,71 +319,71 @@
 		pattern:  "./c/*/*.ext",
 		excludes: []string{"./c/*/f.ext"},
 		matches:  []string{"c/g/g.ext"},
-		dirs:     []string{"c", "c/f", "c/g", "c/h"},
+		deps:     []string{"c", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern:  "c/*/*.ext",
 		excludes: []string{"./c/*/f.ext"},
 		matches:  []string{"c/g/g.ext"},
-		dirs:     []string{"c", "c/f", "c/g", "c/h"},
+		deps:     []string{"c", "c/f", "c/g", "c/h"},
 	},
 	{
 		pattern:  "./c/*/*.ext",
 		excludes: []string{"c/*/f.ext"},
 		matches:  []string{"c/g/g.ext"},
-		dirs:     []string{"c", "c/f", "c/g", "c/h"},
+		deps:     []string{"c", "c/f", "c/g", "c/h"},
 	},
 
 	// non-existant non-wild path tests
 	{
 		pattern: "d/*",
 		matches: nil,
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 	{
 		pattern: "d",
 		matches: nil,
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 	{
 		pattern: "a/d/*",
 		matches: nil,
-		dirs:    []string{"a"},
+		deps:    []string{"a"},
 	},
 	{
 		pattern: "a/d",
 		matches: nil,
-		dirs:    []string{"a"},
+		deps:    []string{"a"},
 	},
 	{
 		pattern: "a/a/d/*",
 		matches: nil,
-		dirs:    []string{"a/a"},
+		deps:    []string{"a/a"},
 	},
 	{
 		pattern: "a/a/d",
 		matches: nil,
-		dirs:    []string{"a/a"},
+		deps:    []string{"a/a"},
 	},
 	{
 		pattern: "a/d/a/*",
 		matches: nil,
-		dirs:    []string{"a"},
+		deps:    []string{"a"},
 	},
 	{
 		pattern: "a/d/a",
 		matches: nil,
-		dirs:    []string{"a"},
+		deps:    []string{"a"},
 	},
 	{
 		pattern: "a/d/a/*/a",
 		matches: nil,
-		dirs:    []string{"a"},
+		deps:    []string{"a"},
 	},
 	{
 		pattern: "a/d/a/**/a",
 		matches: nil,
-		dirs:    []string{"a"},
+		deps:    []string{"a"},
 	},
 
 	// recursive exclude error tests
@@ -422,22 +422,22 @@
 	{
 		pattern: ".test/*",
 		matches: []string{".test/a"},
-		dirs:    []string{".test"},
+		deps:    []string{".test"},
 	},
 	{
 		pattern: ".t*/a",
 		matches: []string{".test/a"},
-		dirs:    []string{".", ".test"},
+		deps:    []string{".", ".test"},
 	},
 	{
 		pattern: ".*/.*",
 		matches: []string{".test/.ing"},
-		dirs:    []string{".", ".test"},
+		deps:    []string{".", ".test"},
 	},
 	{
 		pattern: ".t*",
 		matches: []string{".test", ".testing"},
-		dirs:    []string{"."},
+		deps:    []string{"."},
 	},
 }
 
@@ -445,7 +445,7 @@
 	os.Chdir("testdata")
 	defer os.Chdir("..")
 	for _, testCase := range globTestCases {
-		matches, dirs, err := Glob(testCase.pattern, testCase.excludes)
+		matches, deps, err := Glob(testCase.pattern, testCase.excludes)
 		if err != testCase.err {
 			t.Errorf(" pattern: %q", testCase.pattern)
 			if testCase.excludes != nil {
@@ -464,14 +464,14 @@
 			t.Errorf("     got: %#v", matches)
 			t.Errorf("expected: %#v", testCase.matches)
 		}
-		if !reflect.DeepEqual(dirs, testCase.dirs) {
-			t.Errorf("incorrect dirs list:")
+		if !reflect.DeepEqual(deps, testCase.deps) {
+			t.Errorf("incorrect deps list:")
 			t.Errorf(" pattern: %q", testCase.pattern)
 			if testCase.excludes != nil {
 				t.Errorf("excludes: %q", testCase.excludes)
 			}
-			t.Errorf("     got: %#v", dirs)
-			t.Errorf("expected: %#v", testCase.dirs)
+			t.Errorf("     got: %#v", deps)
+			t.Errorf("expected: %#v", testCase.deps)
 		}
 	}
 }