Jenkins 13493 - Automatic git cache maintenance

pom.xml

@@ -147,6 +147,11 @@
       <artifactId>mailer</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.jenkins.plugins</groupId>
+      <artifactId>data-tables-api</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>
       <artifactId>junit</artifactId>

src/main/java/hudson/plugins/git/GitSCM.java

@@ -302,7 +302,7 @@ public Object readResolve() throws IOException, GitException {
         if (remoteRepositories != null && userRemoteConfigs == null) {
             userRemoteConfigs = new ArrayList<>();
             for(RemoteConfig cfg : remoteRepositories) {
-                // converted as in config.jelly
+                // converted as in index.jelly
                 String url = "";
                 if (cfg.getURIs().size() > 0 && cfg.getURIs().get(0) != null)
                     url = cfg.getURIs().get(0).toPrivateString();

src/main/java/jenkins/plugins/git/AbstractGitSCMSource.java

@@ -168,6 +168,22 @@
 
     private static final Logger LOGGER = Logger.getLogger(AbstractGitSCMSource.class.getName());
 
+    static Set<String> cacheEntries = new HashSet<>();
+
+    static {
+        Jenkins jenkins = Jenkins.getInstanceOrNull();
+        if(jenkins != null){
+            File[] caches = new File(jenkins.getRootDir(),"caches").listFiles();
+            if(caches != null) {
+                for (File cache : caches) {
+                    String cacheEntry = cache.getName();
+                    cacheEntries.add(cacheEntry);
+                }
+            }
+            LOGGER.log(Level.FINE,"Caches on Jenkins controller " + cacheEntries);
+        }
+    }
+
     public AbstractGitSCMSource() {
     }
 
@@ -1284,6 +1300,10 @@
         return getCacheEntry(getRemote());
     }
 
+    protected static Set<String> getCacheEntries(){
+        return cacheEntries;
+    }
+
     protected static File getCacheDir(String cacheEntry) {
         return getCacheDir(cacheEntry, true);
     }
@@ -1298,7 +1318,9 @@
         if (!cacheDir.isDirectory()) {
             if (createDirectory) {
                 boolean ok = cacheDir.mkdirs();
-                if (!ok) {
+                if(ok) {
+                    cacheEntries.add(cacheEntry);
+                }else{
                     LOGGER.log(Level.WARNING, "Failed mkdirs of {0}", cacheDir);
                 }
             } else {

src/main/java/jenkins/plugins/git/maintenance/Cron.java

@@ -0,0 +1,42 @@
+package jenkins.plugins.git.maintenance;
+
+import hudson.Extension;
+import hudson.model.PeriodicWork;
+
+import java.util.Calendar;
+import java.util.concurrent.TimeUnit;
+
+@Extension
+public class Cron extends PeriodicWork {
+
+    private TaskScheduler taskScheduler;
+
+    @Override
+    public long getInitialDelay(){
+        return MIN - (Calendar.getInstance().get(Calendar.SECOND) * 1000L);
+    }
+
+    @Override
+    public long getRecurrencePeriod() {
+        return TimeUnit.MINUTES.toMillis(1);
+    }
+
+    @Override
+    protected void doRun(){
+        scheduleMaintenanceTask();
+    }
+
+    void terminateMaintenanceTaskExecution(){
+        if(taskScheduler != null){
+            taskScheduler.terminateMaintenanceTaskExecution();
+        }
+    }
+
+    private void scheduleMaintenanceTask(){
+        if(taskScheduler == null){
+            taskScheduler = new TaskScheduler();
+        }
+
+        taskScheduler.scheduleTasks();
+    }
+}

src/main/java/jenkins/plugins/git/maintenance/GitMaintenanceSCM.java

@@ -0,0 +1,93 @@
+package jenkins.plugins.git.maintenance;
+
+import jenkins.model.Jenkins;
+import jenkins.plugins.git.AbstractGitSCMSource;
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * GitMaintenanceSCM is responsible for fetching all caches along with locks on Jenkins controller. It extends {@link AbstractGitSCMSource}.
+ *
+ * @author Hrushikesh Rao
+ */
+public class GitMaintenanceSCM extends AbstractGitSCMSource {
+
+    String remote;
+
+    private static Logger LOGGER = Logger.getLogger(GitMaintenanceSCM.class.getName());
+    protected GitMaintenanceSCM(String remote){
+        this.remote = remote;
+    }
+
+    /**
+     * Stores the File object and lock for cache.
+     */
+    static class Cache {
+
+        File cache;
+        Lock lock;
+        Cache(File cache, Lock lock){
+            this.cache = cache;
+            this.lock = lock;
+        }
+
+        /**
+         * Return the File object of a cache.
+         * @return File object of a cache.
+         */
+        public File getCacheFile(){
+            return cache;
+        }
+
+        /**
+         * Returns the lock for a cache.
+         *
+         * @return lock for a cache.
+         */
+        public Lock getLock(){
+            return lock;
+        }
+
+    }
+    @Override
+    public String getCredentialsId() {
+        return null;
+    }
+
+    @Override
+    public String getRemote() {
+        return remote;
+    }
+
+    /**
+     * Returns a list of {@link Cache}.
+     * @return A list of {@link Cache}.
+     */
+    public static List<Cache> getCaches(){
+            Jenkins jenkins = Jenkins.getInstanceOrNull();
+
+            List<Cache> caches = new ArrayList<>();
+            if (jenkins == null){
+                LOGGER.log(Level.WARNING,"Internal error. Couldn't get Jenkins instance.");
+                return caches;
+            }
+
+            for (String cacheEntry : getCacheEntries()) {
+                File cacheDir = getCacheDir(cacheEntry,false);
+                Lock cacheLock = getCacheLock(cacheEntry);
+
+                // skip caches size less than 10 mb
+                if(FileUtils.sizeOfDirectory(cacheDir) < 10000000)continue;
+                LOGGER.log(Level.FINE,"Cache Entry " + cacheEntry);
+                caches.add(new Cache(cacheDir,cacheLock));
+            }
+
+            return caches;
+    }
+}

src/main/java/jenkins/plugins/git/maintenance/Logs/CacheRecord.java

@@ -0,0 +1,116 @@
+package jenkins.plugins.git.maintenance.Logs;
+
+import jenkins.plugins.git.maintenance.TaskType;
+import org.apache.commons.lang.time.DurationFormatUtils;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+public class CacheRecord {
+    String repoName;
+    long repoSize;
+    String maintenanceType;
+    long timeOfExecution;
+    boolean executionStatus;
+    long executionDuration;
+
+    Map<String,List<CacheRecord>> maintenanceData;
+
+
+    // This is to create a new Cache Record when cache is not present.
+    public CacheRecord(String repoName,String maintenanceType){
+        this.repoName = repoName;
+        this.maintenanceType = maintenanceType;
+        maintenanceData = new HashMap<>();
+
+        for(TaskType taskType : TaskType.values()){
+            maintenanceData.put(taskType.getTaskName(),new LinkedList<>());
+        }
+    }
+
+    // This is to add maintenance data to existing Cache Record
+    public CacheRecord(CacheRecord cacheRecord){
+        setExecutionDuration(cacheRecord.executionDuration);
+        setExecutionStatus(cacheRecord.getExecutionStatus());
+        setRepoSize(cacheRecord.getRepoSize());
+        setTimeOfExecution(cacheRecord.timeOfExecution);
+        setMaintenanceType(cacheRecord.getMaintenanceType());
+    }
+
+
+    public String getRepoName() {
+        return repoName;
+    }
+
+    public long getRepoSize() {
+        return repoSize;
+    }
+    public void setRepoSize(long repoSize) {
+        this.repoSize = repoSize;
+    }
+
+    public String getMaintenanceType() {
+        return maintenanceType;
+    }
+
+    public void setMaintenanceType(String maintenanceType){
+        this.maintenanceType = maintenanceType;
+    }
+
+    public String getTimeOfExecution() {
+        Date date = new Date(timeOfExecution * 1000L);
+        SimpleDateFormat sdf = new SimpleDateFormat("dd-MM-yyyy HH:mm");
+        return sdf.format(date);
+    }
+
+    public void setTimeOfExecution(long timeOfExecution) {
+        this.timeOfExecution = timeOfExecution;
+    }
+
+    public boolean getExecutionStatus() {
+        return executionStatus;
+    }
+
+    public void setExecutionStatus(boolean executionStatus) {
+        this.executionStatus = executionStatus;
+    }
+
+    public String getExecutionDuration() {
+        return DurationFormatUtils.formatDuration(executionDuration,"mmm:ss:SSS");
+    }
+
+    public void setExecutionDuration(long executionDuration) {
+        this.executionDuration = executionDuration;
+    }
+
+    public void insertMaintenanceData(CacheRecord record){
+        if(record != null && maintenanceData != null) {
+            List<CacheRecord> list = maintenanceData.get(record.getMaintenanceType());
+            if(list != null) {
+                list.add(0,record);
+                // Maximum storage of 5 Maintenance Records per Cache.
+                if (list.size() > 5)
+                    list.remove(list.size()-1);
+            }
+        }
+    }
+
+    public List<CacheRecord> getAllMaintenanceRecordsForSingleCache(){
+        List<CacheRecord> maintenanceData = new ArrayList<>();
+
+        for(Map.Entry<String,List<CacheRecord>> entry : this.maintenanceData.entrySet()){
+            maintenanceData.addAll(entry.getValue());
+        }
+
+        Collections.sort(maintenanceData,(o1,o2) -> (int) (o2.timeOfExecution - o1.timeOfExecution));
+
+        return maintenanceData;
+    }
+
+}

src/main/java/jenkins/plugins/git/maintenance/Logs/RecordList.java

@@ -0,0 +1,77 @@
+package jenkins.plugins.git.maintenance.Logs;
+
+import java.util.*;
+
+public class RecordList {
+    List<CacheRecord> maintenanceRecords;
+    Set<String> cacheSet;
+
+    public RecordList(){
+        maintenanceRecords = new ArrayList<>();
+        cacheSet = new HashSet<>();
+    }
+
+    List<CacheRecord> getMaintenanceRecords(){
+        return new ArrayList<>(maintenanceRecords);
+    }
+
+    void addRecord(CacheRecord cacheRecord){
+        String repoName = cacheRecord.getRepoName();
+        Set<String> cacheSet = getCacheSet();
+        if(cacheSet.contains(repoName)){
+            // adding record to existing cache list
+            Iterator<CacheRecord> itr = maintenanceRecords.iterator();
+
+            CacheRecord record;
+            while(itr.hasNext()){
+                record = itr.next();
+                if(record.getRepoName().equals(repoName)){
+
+                    // To not lose data of the first maintenance task
+                    if(record.getAllMaintenanceRecordsForSingleCache().size() == 0){
+                        CacheRecord oldCacheRecord = new CacheRecord(record);
+                        record.insertMaintenanceData(oldCacheRecord);
+                    }
+                    CacheRecord childCacheRecord = new CacheRecord(cacheRecord);
+
+                    record.insertMaintenanceData(childCacheRecord);
+
+                    // Updates the Top most Cache with fresh data
+                    record.setTimeOfExecution(childCacheRecord.timeOfExecution);
+                    record.setExecutionStatus(childCacheRecord.getExecutionStatus());
+                    record.setRepoSize(childCacheRecord.getRepoSize());
+                    record.setMaintenanceType(childCacheRecord.getMaintenanceType());
+                    record.setExecutionDuration(childCacheRecord.executionDuration);
+
+                    break;
+                }
+            }
+            return;
+        }
+
+        // Creates a new Cache Entry and adds the data.
+        maintenanceRecords.add(0,cacheRecord);
+        cacheSet.add(repoName);
+    }
+
+    List<CacheRecord> getAllMaintenanceRecordsForSingleCache(String cacheName) {
+        List<CacheRecord> allRecords = null;
+
+        Iterator<CacheRecord> itr = maintenanceRecords.iterator();
+
+        CacheRecord record;
+        while(itr.hasNext()){
+            record = itr.next();
+            if(record.getRepoName().equals(cacheName)){
+                allRecords = record.getAllMaintenanceRecordsForSingleCache();
+                break;
+            }
+        }
+
+        return allRecords;
+    }
+
+    Set<String> getCacheSet(){
+        return this.cacheSet;
+    }
+}