Corrected list_bucket to search in stat cache during creating new file

src/cache.cpp

@@ -398,6 +398,13 @@ bool StatCache::AddStat(const std::string& key, const headers_t& meta, bool forc
             DelSymlink(key.c_str(), AutoLock::ALREADY_LOCKED);
         }
     }
+
+    // If no_truncate flag is set, set file name to notruncate_file_cache
+    //
+    if(no_truncate){
+        AddNotruncateCache(key);
+    }
+
     return true;
 }
 
@@ -509,10 +516,18 @@ void StatCache::ChangeNoTruncateFlag(const std::string& key, bool no_truncate)
     if(stat_cache.end() != iter){
         stat_cache_entry* ent = &iter->second;
         if(no_truncate){
+            if(0L == ent->notruncate){
+                // need to add no truncate cache.
+                AddNotruncateCache(key);
+            }
             ++(ent->notruncate);
         }else{
             if(0L < ent->notruncate){
                 --(ent->notruncate);
+                if(0L == ent->notruncate){
+                    // need to delete from no truncate cache.
+                    DelNotruncateCache(key);
+                }
             }
         }
     }
@@ -587,6 +602,12 @@ bool StatCache::DelStat(const char* key, AutoLock::Type locktype)
 
     stat_cache_t::iterator iter;
     if(stat_cache.end() != (iter = stat_cache.find(key))){
+        // If found stat cache has notruncate flag, remove it from notruncate_file_cache.
+        //
+        const stat_cache_entry* ent = &iter->second;
+        if(0L < ent->notruncate){
+            DelNotruncateCache(key);
+        }
         stat_cache.erase(iter);
     }
     if(0 < strlen(key) && 0 != strcmp(key, "/")){
@@ -599,6 +620,12 @@ bool StatCache::DelStat(const char* key, AutoLock::Type locktype)
             strpath += "/";
         }
         if(stat_cache.end() != (iter = stat_cache.find(strpath))){
+            // If found stat cache has notruncate flag, remove it from notruncate_file_cache.
+            //
+            const stat_cache_entry* ent = &iter->second;
+            if(0L < ent->notruncate){
+                DelNotruncateCache(strpath);
+            }
             stat_cache.erase(iter);
         }
     }
@@ -746,6 +773,116 @@ bool StatCache::DelSymlink(const char* key, AutoLock::Type locktype)
     return true;
 }
 
+// [NOTE]
+// Need to lock StatCache::stat_cache_lock before calling this method.
+//
+bool StatCache::AddNotruncateCache(const std::string& key)
+{
+    if(key.empty() || '/' == *key.rbegin()){
+        return false;
+    }
+
+    std::string parentdir = mydirname(key);
+    std::string filename  = mybasename(key);
+    if(parentdir.empty() || filename.empty()){
+        return false;
+    }
+    parentdir += '/';       // directory path must be '/' termination.
+
+    notruncate_dir_map_t::iterator iter = notruncate_file_cache.find(parentdir);
+    if(iter == notruncate_file_cache.end()){
+        // add new list
+        notruncate_filelist_t list;
+        list.push_back(filename);
+        notruncate_file_cache[parentdir] = list;
+    }else{
+        // add filename to existed list
+        notruncate_filelist_t& filelist = iter->second;
+        notruncate_filelist_t::const_iterator fiter = std::find(filelist.begin(), filelist.end(), filename);
+        if(fiter == filelist.end()){
+            filelist.push_back(filename);
+        }
+    }
+    return true;
+}
+
+// [NOTE]
+// Need to lock StatCache::stat_cache_lock before calling this method.
+//
+bool StatCache::DelNotruncateCache(const std::string& key)
+{
+    if(key.empty() || '/' == *key.rbegin()){
+        return false;
+    }
+
+    std::string parentdir = mydirname(key);
+    std::string filename  = mybasename(key);
+    if(parentdir.empty() || filename.empty()){
+        return false;
+    }
+    parentdir += '/';       // directory path must be '/' termination.
+
+    notruncate_dir_map_t::iterator iter = notruncate_file_cache.find(parentdir);
+    if(iter != notruncate_file_cache.end()){
+        // found directory in map
+        notruncate_filelist_t& filelist = iter->second;
+        notruncate_filelist_t::iterator fiter = std::find(filelist.begin(), filelist.end(), filename);
+        if(fiter != filelist.end()){
+            // found filename in directory file list
+            filelist.erase(fiter);
+            if(filelist.empty()){
+                notruncate_file_cache.erase(parentdir);
+            }
+        }
+    }
+    return true;
+}
+
+// [Background]
+// When s3fs creates a new file, the file does not exist until the file contents
+// are uploaded.(because it doesn't create a 0 byte file)
+// From the time this file is created(opened) until it is uploaded(flush), it
+// will have a Stat cache with the No truncate flag added.
+// This avoids file not existing errors in operations such as chmod and utimens
+// that occur in the short period before file upload.
+// Besides this, we also need to support readdir(list_bucket), this method is
+// called to maintain the cache for readdir and return its value.
+//
+// [NOTE]
+// Add the file names under parentdir to the list.
+// However, if the same file name exists in the list, it will not be added.
+// parentdir must be terminated with a '/'.
+//
+bool StatCache::GetNotruncateCache(const std::string& parentdir, notruncate_filelist_t& list)
+{
+    if(parentdir.empty()){
+        return false;
+    }
+
+    std::string dirpath = parentdir;
+    if('/' != *dirpath.rbegin()){
+        dirpath += '/';
+    }
+
+    AutoLock lock(&StatCache::stat_cache_lock);
+
+    notruncate_dir_map_t::iterator iter = notruncate_file_cache.find(dirpath);
+    if(iter == notruncate_file_cache.end()){
+        // nothing to add
+        return true;
+    }
+
+    // found directory in map
+    const notruncate_filelist_t& filelist = iter->second;
+    for(notruncate_filelist_t::const_iterator fiter = filelist.begin(); fiter != filelist.end(); ++fiter){
+        if(list.end() == std::find(list.begin(), list.end(), *fiter)){
+           // found notuncate file that does not exist in the list, so add it.
+           list.push_back(*fiter);
+        }
+    }
+    return true;
+}
+
 //-------------------------------------------------------------------
 // Functions
 //-------------------------------------------------------------------

src/cache.h

@@ -69,6 +69,12 @@ struct symlink_cache_entry {
 
 typedef std::map<std::string, symlink_cache_entry> symlink_cache_t;
 
+//
+// Typedefs for No truncate file name cache
+//
+typedef std::vector<std::string> notruncate_filelist_t;                    // untruncated file name list in dir
+typedef std::map<std::string, notruncate_filelist_t> notruncate_dir_map_t; // key is parent dir path
+
 //-------------------------------------------------------------------
 // Class StatCache
 //-------------------------------------------------------------------
@@ -93,6 +99,7 @@ class StatCache
         unsigned long          CacheSize;
         bool                   IsCacheNoObject;
         symlink_cache_t        symlink_cache;
+        notruncate_dir_map_t   notruncate_file_cache;
 
     private:
         StatCache();
@@ -105,6 +112,9 @@ class StatCache
         // Truncate symbolic link cache
         bool TruncateSymlink();
 
+        bool AddNotruncateCache(const std::string& key);
+        bool DelNotruncateCache(const std::string& key);
+
     public:
         // Reference singleton
         static StatCache* getStatCacheData()
@@ -182,6 +192,9 @@ class StatCache
         bool GetSymlink(const std::string& key, std::string& value);
         bool AddSymlink(const std::string& key, const std::string& value);
         bool DelSymlink(const char* key, AutoLock::Type locktype = AutoLock::NONE);
+
+        // Cache for Notruncate file
+        bool GetNotruncateCache(const std::string& parentdir, notruncate_filelist_t& list);
 };
 
 //-------------------------------------------------------------------

src/s3fs.cpp

@@ -1775,8 +1775,9 @@ static int rename_directory(const char* from, const char* to)
         S3FS_PRN_ERR("list_bucket returns error.");
         return result; 
     }
-    head.GetNameList(headlist);                       // get name without "/".
-    S3ObjList::MakeHierarchizedList(headlist, false); // add hierarchized dir.
+    head.GetNameList(headlist);                                             // get name without "/".
+    StatCache::getStatCacheData()->GetNotruncateCache(basepath, headlist);  // Add notruncate file name from stat cache
+    S3ObjList::MakeHierarchizedList(headlist, false);                       // add hierarchized dir.
 
     s3obj_list_t::const_iterator liter;
     for(liter = headlist.begin(); headlist.end() != liter; ++liter){
@@ -3304,7 +3305,8 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
     S3FS_PRN_INFO1("[path=%s][list=%zu]", path, headlist.size());
 
     // Make base path list.
-    head.GetNameList(headlist, true, false);  // get name with "/".
+    head.GetNameList(headlist, true, false);                                        // get name with "/".
+    StatCache::getStatCacheData()->GetNotruncateCache(std::string(path), headlist); // Add notruncate file name from stat cache
 
     // Initialize S3fsMultiCurl
     curlmulti.SetSuccessCallback(multi_head_callback);