feat: index ignored paths

packages/fuzzy-finder/lib/load-paths-handler.js

@@ -19,6 +19,7 @@ const realRgPath = rgPath.replace(/\bapp\.asar\b/, 'app.asar.unpacked')
 const MaxConcurrentCrawls = Math.min(Math.max(os.cpus().length - 1, 8), 1)
 
 const trackedPaths = new Set()
+const ignoredPaths = new Set()
 
 class PathLoader {
   constructor (rootPath, options) {
@@ -29,6 +30,7 @@ class PathLoader {
     this.useRipGrep = options.useRipGrep
     this.indexIgnoredPaths = options.indexIgnoredPaths
     this.paths = []
+    this.ignoredPaths = []
     this.inodes = new Set()
     this.repo = null
     if (this.ignoreVcsIgnores && !this.useRipGrep) {
@@ -41,7 +43,11 @@ class PathLoader {
 
   load (done) {
     if (this.useRipGrep) {
-      this.loadFromRipGrep().then(done)
+      // first, load tracked paths and populate the set of tracked paths
+      // then, load all paths (tracked and not), using the above set to differentiate
+      this.loadFromRipGrep()
+        .then(() => this.indexIgnoredPaths && this.loadFromRipGrep({loadIgnoredPaths: true}))
+        .then(done)
 
       return
     }
@@ -53,20 +59,22 @@ class PathLoader {
     })
   }
 
-  async loadFromRipGrep () {
+  async loadFromRipGrep (options = {}) {
     return new Promise((resolve) => {
       const args = ['--files', '--hidden', '--sort', 'path']
 
-      if (!this.ignoreVcsIgnores) {
+      if (!this.ignoreVcsIgnores || options.loadIgnoredPaths) {
         args.push('--no-ignore')
       }
 
       if (this.traverseSymlinkDirectories) {
         args.push('--follow')
       }
 
-      for (let ignoredName of this.ignoredNames) {
-        args.push('-g', '!' + ignoredName.pattern)
+      if (! options.loadIgnoredPaths) {
+          for (let ignoredName of this.ignoredNames) {
+            args.push('-g', '!' + ignoredName.pattern)
+          }
       }
 
       if (this.ignoreVcsIgnores) {
@@ -83,7 +91,11 @@ class PathLoader {
 
         for (const file of files) {
           let loadedPath = path.join(this.rootPath, file)
-            this.trackedPathLoaded(loadedPath, null)
+          if (options.loadIgnoredPaths) {
+            this.ignoredPathLoaded(loadedPath)
+          } else {
+            this.trackedPathLoaded(loadedPath)
+          }
         }
       })
       result.stderr.on('data', () => {
@@ -116,16 +128,36 @@ class PathLoader {
     if (this.paths.length === PathsChunkSize) {
       this.flushPaths()
     }
+
+    done && done()
+  }
+
+  ignoredPathLoaded (loadedPath, done) {
+    if (trackedPaths.has(loadedPath)) {
+        return
+    }
+
+    if (!ignoredPaths.has(loadedPath)) {
+      ignoredPaths.add(loadedPath)
+      this.ignoredPaths.push(loadedPath)
+    }
+
+    if (this.ignoredPaths.length === PathsChunkSize) {
+      this.flushPaths()
+    }
+
     done && done()
   }
 
   flushPaths () {
-    emit('load-paths:paths-found', this.paths)
+    emit('load-paths:paths-found', {paths: this.paths, ignoredPaths: this.ignoredPaths})
     this.paths = []
+    this.ignoredPaths = []
   }
 
   loadPath (pathToLoad, root, done) {
-    if (this.isIgnored(pathToLoad) && !root) return done()
+    const isIgnored = this.isIgnored(pathToLoad)
+    if (isIgnored && !this.indexIgnoredPaths && !root) return done()
 
     fs.lstat(pathToLoad, (error, stats) => {
       if (error != null) { return done() }
@@ -139,7 +171,13 @@ class PathLoader {
           }
 
           if (stats.isFile()) {
-            this.trackedPathLoaded(pathToLoad, done)
+            if (!isIgnored ) {
+              this.trackedPathLoaded(pathToLoad, done)
+            } else if (this.indexIgnoredPaths) {
+              this.ignoredPathLoaded(pathToLoad, done)
+            } else {
+              done()
+            }
           } else if (stats.isDirectory()) {
             if (this.traverseSymlinkDirectories) {
               this.loadFolder(pathToLoad, done)
@@ -153,9 +191,22 @@ class PathLoader {
       } else {
         this.inodes.add(stats.ino)
         if (stats.isDirectory()) {
-          this.loadFolder(pathToLoad, done)
+          // descend into the .git dir only if we're including ignored paths
+          // FIXME this it not correct if the repo dir is non-default
+          // FIXME / is not platform agnostic
+          if (!pathToLoad.match(/(^|\/)\.git/) || !this.ignoreVcsIgnores) {
+            this.loadFolder(pathToLoad, done)
+          } else {
+            done()
+          }
         } else if (stats.isFile()) {
-          this.trackedPathLoaded(pathToLoad, done)
+          if (!isIgnored) {
+            this.trackedPathLoaded(pathToLoad, done)
+          } else if (this.indexIgnoredPaths) {
+            this.ignoredPathLoaded(pathToLoad, done)
+          } else {
+            done()
+          }
         } else {
           done()
         }

packages/fuzzy-finder/lib/main.js

@@ -48,6 +48,7 @@ module.exports = {
       this.gitStatusView = null
     }
     this.projectPaths = null
+    this.ignoredPaths = null
     this.stopLoadPathsTask()
     this.active = false
   },
@@ -82,8 +83,9 @@ module.exports = {
 
     if (this.projectView == null) {
       const ProjectView = require('./project-view')
-      this.projectView = new ProjectView(this.projectPaths)
+      this.projectView = new ProjectView(this.projectPaths, this.ignoredPaths)
       this.projectPaths = null
+      this.ignoredPaths = null
       if (this.teletypeService) {
         this.projectView.setTeletypeService(this.teletypeService)
       }
@@ -117,11 +119,13 @@ module.exports = {
     if (atom.project.getPaths().length === 0) return
 
     const PathLoader = require('./path-loader')
-    this.loadPathsTask = PathLoader.startTask((projectPaths) => {
+    this.loadPathsTask = PathLoader.startTask((projectPaths, ignoredPaths) => {
       this.projectPaths = projectPaths
+      this.ignoredPaths = ignoredPaths
     })
     this.projectPathsSubscription = atom.project.onDidChangePaths(() => {
       this.projectPaths = null
+      this.ignoredPaths = null
       this.stopLoadPathsTask()
     })
   },

packages/fuzzy-finder/lib/path-loader.js

@@ -4,6 +4,7 @@ const {Task} = require('atom')
 module.exports = {
   startTask (callback) {
     const results = []
+    const ignoredResults = []
     const taskPath = require.resolve('./load-paths-handler')
     const followSymlinks = atom.config.get('core.followSymlinks')
     let ignoredNames = atom.config.get('fuzzy-finder.ignoredNames') || []
@@ -23,13 +24,14 @@ module.exports = {
           useRipGrep,
           indexIgnoredPaths
       },
-      () => callback(results)
+      () => callback(results, ignoredResults)
     )
 
     task.on('load-paths:paths-found',
       (paths) => {
-        paths = paths || []
-        results.push(...paths)
+        paths = paths || {paths: [], ignoredPaths: []}
+        results.push(...paths.paths)
+        ignoredResults.push(...paths.ignoredPaths)
       }
     )
 

packages/fuzzy-finder/lib/project-view.js

@@ -6,10 +6,12 @@ const PathLoader = require('./path-loader')
 
 module.exports =
 class ProjectView extends FuzzyFinderView {
-  constructor (paths) {
+  constructor (paths, ignoredPaths) {
     super()
     this.disposables = new CompositeDisposable()
     this.paths = paths
+    this.ignoredPaths = ignoredPaths
+    // if no paths were passed in, then we should try to reload them
     this.reloadPaths = !this.paths || this.paths.length === 0
     this.reloadAfterFirstLoad = false
 
@@ -30,8 +32,10 @@ class ProjectView extends FuzzyFinderView {
     this.disposables.add(atom.config.onDidChange('core.ignoredNames', () => { this.reloadPaths = true }))
     this.disposables.add(atom.config.onDidChange('core.excludeVcsIgnoredPaths', () => { this.reloadPaths = true }))
     this.disposables.add(atom.project.onDidChangePaths(() => {
+      // if a project path was changed/added/removed, clear our list and reindex
       this.reloadPaths = true
       this.paths = null
+      this.ignoredPaths = null
     }))
 
     if (!this.reloadPaths) {
@@ -74,7 +78,8 @@ class ProjectView extends FuzzyFinderView {
     })
 
     const localItems = this.projectRelativePathsForFilePaths(this.paths || [])
-    await this.setItems(remoteItems.concat(localItems))
+    const localIgnoredItems = this.projectRelativePathsForFilePaths(this.ignoredPaths || [])
+    await this.setItems(remoteItems.concat(localItems), localIgnoredItems)
   }
 
   async reloadPathsIfNeeded () {
@@ -114,7 +119,8 @@ class ProjectView extends FuzzyFinderView {
         if (task) {
           let pathsFound = 0
           task.on('load-paths:paths-found', (paths) => {
-            pathsFound += paths.length
+            paths = paths || {paths: [], ignoredPaths: []}
+            pathsFound += paths.paths.length + paths.ignoredPaths.length
             this.selectListView.update({loadingMessage: 'Indexing project\u2026', infoMessage: null, loadingBadge: humanize.intComma(pathsFound)})
           })
         }
@@ -178,8 +184,9 @@ class ProjectView extends FuzzyFinderView {
       this.loadPathsTask.terminate()
     }
 
-    this.loadPathsTask = PathLoader.startTask((paths) => {
+    this.loadPathsTask = PathLoader.startTask((paths, ignoredPaths) => {
       this.paths = paths
+      this.ignoredPaths = ignoredPaths
       this.reloadPaths = false
       if (fn) {
         fn()

packages/fuzzy-finder/spec/fuzzy-finder-spec.js

@@ -822,21 +822,26 @@ describe('FuzzyFinder', () => {
           })
 
           it('passes the indexed paths into the project view when it is created', () => {
-            const {projectPaths} = fuzzyFinder
+            const {projectPaths, ignoredPaths} = fuzzyFinder
             expect(projectPaths.length).toBe(19)
+            expect(ignoredPaths.length).toBe(0)
+
             projectView = fuzzyFinder.createProjectView()
             expect(projectView.paths).toBe(projectPaths)
+            expect(projectView.ignoredPaths).toBe(ignoredPaths)
             expect(projectView.reloadPaths).toBe(false)
           })
 
           it('busts the cached paths when the project paths change', () => {
             atom.project.setPaths([])
 
-            const {projectPaths} = fuzzyFinder
+            const {projectPaths, ignoredPaths} = fuzzyFinder
             expect(projectPaths).toBe(null)
+            expect(ignoredPaths).toBe(null)
 
             projectView = fuzzyFinder.createProjectView()
             expect(projectView.paths).toBe(null)
+            expect(projectView.ignoredPaths).toBe(null)
             expect(projectView.reloadPaths).toBe(true)
           })
         })
@@ -1612,6 +1617,21 @@ describe('FuzzyFinder', () => {
 
               await waitForPathsToDisplay(projectView)
 
+              expect(projectView.paths.length).toBe(6)
+              expect(projectView.ignoredPaths.length).toBe(0)
+
+              expect(Array.from(projectView.element.querySelectorAll('li')).find(a => a.textContent.includes('ignored.txt'))).not.toBeDefined()
+            })
+
+            it('includes paths that are git ignored when indexIgnoredPaths is true', async () => {
+              atom.config.set('fuzzy-finder.indexIgnoredPaths', true)
+              await projectView.toggle()
+
+              await waitForPathsToDisplay(projectView)
+
+              expect(projectView.paths.length).toBe(6)
+              expect(projectView.ignoredPaths.length).toBe(1)
+
               expect(Array.from(projectView.element.querySelectorAll('li')).find(a => a.textContent.includes('ignored.txt'))).not.toBeDefined()
             })
           })

packages/fuzzy-finder/spec/project-view-spec.js

@@ -20,7 +20,7 @@ describe('ProjectView', () => {
   })
 
   it('includes remote editors when teletype is enabled', async () => {
-    const projectView = new ProjectView([])
+    const projectView = new ProjectView([], [])
 
     const projectPath = fs.realpathSync(temp.mkdirSync())
     const file1Path = path.join(projectPath, 'a')
@@ -49,7 +49,7 @@ describe('ProjectView', () => {
   })
 
   it('shows remote editors even when there is no open project', async () => {
-    const projectView = new ProjectView([])
+    const projectView = new ProjectView([], [])
 
     atom.project.setPaths([])
     projectView.setTeletypeService({
@@ -69,7 +69,7 @@ describe('ProjectView', () => {
   })
 
   it('gracefully defaults to empty list if teletype is unable to provide remote editors', async () => {
-    const projectView = new ProjectView([])
+    const projectView = new ProjectView([], [])
 
     atom.project.setPaths([])
     projectView.setTeletypeService({