diff --git a/pom.xml b/pom.xml index db77f43d95..87b4b335f6 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,11 @@ mailer + + io.jenkins.plugins + data-tables-api + + org.jenkins-ci.plugins junit diff --git a/src/main/java/hudson/plugins/git/GitSCM.java b/src/main/java/hudson/plugins/git/GitSCM.java index ceab12095a..7117609ca6 100644 --- a/src/main/java/hudson/plugins/git/GitSCM.java +++ b/src/main/java/hudson/plugins/git/GitSCM.java @@ -302,7 +302,7 @@ public Object readResolve() throws IOException, GitException { if (remoteRepositories != null && userRemoteConfigs == null) { userRemoteConfigs = new ArrayList<>(); for(RemoteConfig cfg : remoteRepositories) { - // converted as in config.jelly + // converted as in index.jelly String url = ""; if (cfg.getURIs().size() > 0 && cfg.getURIs().get(0) != null) url = cfg.getURIs().get(0).toPrivateString(); diff --git a/src/main/java/jenkins/plugins/git/AbstractGitSCMSource.java b/src/main/java/jenkins/plugins/git/AbstractGitSCMSource.java index 334f292f48..c676f406dd 100644 --- a/src/main/java/jenkins/plugins/git/AbstractGitSCMSource.java +++ b/src/main/java/jenkins/plugins/git/AbstractGitSCMSource.java @@ -168,6 +168,22 @@ public abstract class AbstractGitSCMSource extends SCMSource { private static final Logger LOGGER = Logger.getLogger(AbstractGitSCMSource.class.getName()); + static Set cacheEntries = new HashSet<>(); + + static { + Jenkins jenkins = Jenkins.getInstanceOrNull(); + if(jenkins != null){ + File[] caches = new File(jenkins.getRootDir(),"caches").listFiles(); + if(caches != null) { + for (File cache : caches) { + String cacheEntry = cache.getName(); + cacheEntries.add(cacheEntry); + } + } + LOGGER.log(Level.FINE,"Caches on Jenkins controller " + cacheEntries); + } + } + public AbstractGitSCMSource() { } @@ -1284,6 +1300,10 @@ protected String getCacheEntry() { return getCacheEntry(getRemote()); } + protected static Set getCacheEntries(){ + return cacheEntries; + } + protected static File getCacheDir(String cacheEntry) { return getCacheDir(cacheEntry, true); } @@ -1298,7 +1318,9 @@ protected static File getCacheDir(String cacheEntry, boolean createDirectory) { if (!cacheDir.isDirectory()) { if (createDirectory) { boolean ok = cacheDir.mkdirs(); - if (!ok) { + if(ok) { + cacheEntries.add(cacheEntry); + }else{ LOGGER.log(Level.WARNING, "Failed mkdirs of {0}", cacheDir); } } else { diff --git a/src/main/java/jenkins/plugins/git/maintenance/Cron.java b/src/main/java/jenkins/plugins/git/maintenance/Cron.java new file mode 100644 index 0000000000..889dc6d340 --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/Cron.java @@ -0,0 +1,42 @@ +package jenkins.plugins.git.maintenance; + +import hudson.Extension; +import hudson.model.PeriodicWork; + +import java.util.Calendar; +import java.util.concurrent.TimeUnit; + +@Extension +public class Cron extends PeriodicWork { + + private TaskScheduler taskScheduler; + + @Override + public long getInitialDelay(){ + return MIN - (Calendar.getInstance().get(Calendar.SECOND) * 1000L); + } + + @Override + public long getRecurrencePeriod() { + return TimeUnit.MINUTES.toMillis(1); + } + + @Override + protected void doRun(){ + scheduleMaintenanceTask(); + } + + void terminateMaintenanceTaskExecution(){ + if(taskScheduler != null){ + taskScheduler.terminateMaintenanceTaskExecution(); + } + } + + private void scheduleMaintenanceTask(){ + if(taskScheduler == null){ + taskScheduler = new TaskScheduler(); + } + + taskScheduler.scheduleTasks(); + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/GitMaintenanceSCM.java b/src/main/java/jenkins/plugins/git/maintenance/GitMaintenanceSCM.java new file mode 100644 index 0000000000..701beb1b33 --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/GitMaintenanceSCM.java @@ -0,0 +1,93 @@ +package jenkins.plugins.git.maintenance; + +import jenkins.model.Jenkins; +import jenkins.plugins.git.AbstractGitSCMSource; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * GitMaintenanceSCM is responsible for fetching all caches along with locks on Jenkins controller. It extends {@link AbstractGitSCMSource}. + * + * @author Hrushikesh Rao + */ +public class GitMaintenanceSCM extends AbstractGitSCMSource { + + String remote; + + private static Logger LOGGER = Logger.getLogger(GitMaintenanceSCM.class.getName()); + protected GitMaintenanceSCM(String remote){ + this.remote = remote; + } + + /** + * Stores the File object and lock for cache. + */ + static class Cache { + + File cache; + Lock lock; + Cache(File cache, Lock lock){ + this.cache = cache; + this.lock = lock; + } + + /** + * Return the File object of a cache. + * @return File object of a cache. + */ + public File getCacheFile(){ + return cache; + } + + /** + * Returns the lock for a cache. + * + * @return lock for a cache. + */ + public Lock getLock(){ + return lock; + } + + } + @Override + public String getCredentialsId() { + return null; + } + + @Override + public String getRemote() { + return remote; + } + + /** + * Returns a list of {@link Cache}. + * @return A list of {@link Cache}. + */ + public static List getCaches(){ + Jenkins jenkins = Jenkins.getInstanceOrNull(); + + List caches = new ArrayList<>(); + if (jenkins == null){ + LOGGER.log(Level.WARNING,"Internal error. Couldn't get Jenkins instance."); + return caches; + } + + for (String cacheEntry : getCacheEntries()) { + File cacheDir = getCacheDir(cacheEntry,false); + Lock cacheLock = getCacheLock(cacheEntry); + + // skip caches size less than 10 mb + if(FileUtils.sizeOfDirectory(cacheDir) < 10000000)continue; + LOGGER.log(Level.FINE,"Cache Entry " + cacheEntry); + caches.add(new Cache(cacheDir,cacheLock)); + } + + return caches; + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/Logs/CacheRecord.java b/src/main/java/jenkins/plugins/git/maintenance/Logs/CacheRecord.java new file mode 100644 index 0000000000..8b06f4874e --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/Logs/CacheRecord.java @@ -0,0 +1,116 @@ +package jenkins.plugins.git.maintenance.Logs; + +import jenkins.plugins.git.maintenance.TaskType; +import org.apache.commons.lang.time.DurationFormatUtils; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +public class CacheRecord { + String repoName; + long repoSize; + String maintenanceType; + long timeOfExecution; + boolean executionStatus; + long executionDuration; + + Map> maintenanceData; + + + // This is to create a new Cache Record when cache is not present. + public CacheRecord(String repoName,String maintenanceType){ + this.repoName = repoName; + this.maintenanceType = maintenanceType; + maintenanceData = new HashMap<>(); + + for(TaskType taskType : TaskType.values()){ + maintenanceData.put(taskType.getTaskName(),new LinkedList<>()); + } + } + + // This is to add maintenance data to existing Cache Record + public CacheRecord(CacheRecord cacheRecord){ + setExecutionDuration(cacheRecord.executionDuration); + setExecutionStatus(cacheRecord.getExecutionStatus()); + setRepoSize(cacheRecord.getRepoSize()); + setTimeOfExecution(cacheRecord.timeOfExecution); + setMaintenanceType(cacheRecord.getMaintenanceType()); + } + + + public String getRepoName() { + return repoName; + } + + public long getRepoSize() { + return repoSize; + } + public void setRepoSize(long repoSize) { + this.repoSize = repoSize; + } + + public String getMaintenanceType() { + return maintenanceType; + } + + public void setMaintenanceType(String maintenanceType){ + this.maintenanceType = maintenanceType; + } + + public String getTimeOfExecution() { + Date date = new Date(timeOfExecution * 1000L); + SimpleDateFormat sdf = new SimpleDateFormat("dd-MM-yyyy HH:mm"); + return sdf.format(date); + } + + public void setTimeOfExecution(long timeOfExecution) { + this.timeOfExecution = timeOfExecution; + } + + public boolean getExecutionStatus() { + return executionStatus; + } + + public void setExecutionStatus(boolean executionStatus) { + this.executionStatus = executionStatus; + } + + public String getExecutionDuration() { + return DurationFormatUtils.formatDuration(executionDuration,"mmm:ss:SSS"); + } + + public void setExecutionDuration(long executionDuration) { + this.executionDuration = executionDuration; + } + + public void insertMaintenanceData(CacheRecord record){ + if(record != null && maintenanceData != null) { + List list = maintenanceData.get(record.getMaintenanceType()); + if(list != null) { + list.add(0,record); + // Maximum storage of 5 Maintenance Records per Cache. + if (list.size() > 5) + list.remove(list.size()-1); + } + } + } + + public List getAllMaintenanceRecordsForSingleCache(){ + List maintenanceData = new ArrayList<>(); + + for(Map.Entry> entry : this.maintenanceData.entrySet()){ + maintenanceData.addAll(entry.getValue()); + } + + Collections.sort(maintenanceData,(o1,o2) -> (int) (o2.timeOfExecution - o1.timeOfExecution)); + + return maintenanceData; + } + +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/Logs/RecordList.java b/src/main/java/jenkins/plugins/git/maintenance/Logs/RecordList.java new file mode 100644 index 0000000000..a591fa3f8c --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/Logs/RecordList.java @@ -0,0 +1,77 @@ +package jenkins.plugins.git.maintenance.Logs; + +import java.util.*; + +public class RecordList { + List maintenanceRecords; + Set cacheSet; + + public RecordList(){ + maintenanceRecords = new ArrayList<>(); + cacheSet = new HashSet<>(); + } + + List getMaintenanceRecords(){ + return new ArrayList<>(maintenanceRecords); + } + + void addRecord(CacheRecord cacheRecord){ + String repoName = cacheRecord.getRepoName(); + Set cacheSet = getCacheSet(); + if(cacheSet.contains(repoName)){ + // adding record to existing cache list + Iterator itr = maintenanceRecords.iterator(); + + CacheRecord record; + while(itr.hasNext()){ + record = itr.next(); + if(record.getRepoName().equals(repoName)){ + + // To not lose data of the first maintenance task + if(record.getAllMaintenanceRecordsForSingleCache().size() == 0){ + CacheRecord oldCacheRecord = new CacheRecord(record); + record.insertMaintenanceData(oldCacheRecord); + } + CacheRecord childCacheRecord = new CacheRecord(cacheRecord); + + record.insertMaintenanceData(childCacheRecord); + + // Updates the Top most Cache with fresh data + record.setTimeOfExecution(childCacheRecord.timeOfExecution); + record.setExecutionStatus(childCacheRecord.getExecutionStatus()); + record.setRepoSize(childCacheRecord.getRepoSize()); + record.setMaintenanceType(childCacheRecord.getMaintenanceType()); + record.setExecutionDuration(childCacheRecord.executionDuration); + + break; + } + } + return; + } + + // Creates a new Cache Entry and adds the data. + maintenanceRecords.add(0,cacheRecord); + cacheSet.add(repoName); + } + + List getAllMaintenanceRecordsForSingleCache(String cacheName) { + List allRecords = null; + + Iterator itr = maintenanceRecords.iterator(); + + CacheRecord record; + while(itr.hasNext()){ + record = itr.next(); + if(record.getRepoName().equals(cacheName)){ + allRecords = record.getAllMaintenanceRecordsForSingleCache(); + break; + } + } + + return allRecords; + } + + Set getCacheSet(){ + return this.cacheSet; + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/Logs/XmlSerialize.java b/src/main/java/jenkins/plugins/git/maintenance/Logs/XmlSerialize.java new file mode 100644 index 0000000000..acd22c3a72 --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/Logs/XmlSerialize.java @@ -0,0 +1,96 @@ +package jenkins.plugins.git.maintenance.Logs; + +import com.thoughtworks.xstream.XStream; +import com.thoughtworks.xstream.io.xml.DomDriver; +import com.thoughtworks.xstream.security.AnyTypePermission; +import jenkins.model.Jenkins; +import jenkins.plugins.git.maintenance.GitMaintenanceSCM; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class XmlSerialize{ + + XStream xStream; + + File maintenanceRecordsFile; + + RecordList recordList; + + String maintenanceFileName = "maintenanceRecords.xml"; + private static Logger LOGGER = Logger.getLogger(GitMaintenanceSCM.class.getName()); + + public XmlSerialize(){ + this.xStream = new XStream(new DomDriver()); + // Need to change the Permission type. Todo need to read documentation and update security. + this.xStream.addPermission(AnyTypePermission.ANY); + Jenkins jenkins = Jenkins.getInstanceOrNull(); + + if(jenkins != null) { + File rootDir = jenkins.getRootDir(); + this.maintenanceRecordsFile = new File(rootDir.getAbsolutePath(), maintenanceFileName); + } + } + + RecordList fetchMaintenanceData(){ + if(maintenanceRecordsFile == null){ + // Need to log..... + LOGGER.log(Level.FINE,maintenanceFileName + " file path error."); + return null; + } + + // Checks if recordList is loaded from xml. If not loaded, load it. + if(recordList == null) { + try { + RecordList recordList; + if (!maintenanceRecordsFile.exists()) { + recordList = new RecordList(); + LOGGER.log(Level.FINE,maintenanceFileName + " file doesn't exist"); + } else { + byte[] parsedXmlByteArr = Files.readAllBytes(Paths.get(maintenanceRecordsFile.getAbsolutePath())); + String parsedXmlString = new String(parsedXmlByteArr, StandardCharsets.UTF_8); + + xStream.setClassLoader(RecordList.class.getClassLoader()); + recordList = (RecordList) xStream.fromXML(parsedXmlString); + LOGGER.log(Level.FINE,"Maintenance data loaded from " + maintenanceFileName); + } + this.recordList = recordList; + } catch (IOException e) { + LOGGER.log(Level.FINE,"Couldn't load data from " + maintenanceFileName + ". Err: " + e.getMessage()); + } + } + + return this.recordList; + } + + public boolean addMaintenanceRecord(CacheRecord record){ + RecordList recordList = fetchMaintenanceData(); + + if(recordList != null){ + try { + + recordList.addRecord(record); + String xmlData = xStream.toXML(recordList); + Files.write(Paths.get(maintenanceRecordsFile.getAbsolutePath()), xmlData.getBytes(StandardCharsets.UTF_8)); + return true; + }catch (IOException e){ + LOGGER.log(Level.FINE,"Error writing a record to " + maintenanceFileName + ". Err: " + e.getMessage()); + } + } + return false; + } + + public List getMaintenanceRecords(){ + return fetchMaintenanceData().getMaintenanceRecords(); + } + + public List getAllMaintenanceRecordsForSingleCache(String cacheName) { + return fetchMaintenanceData().getAllMaintenanceRecordsForSingleCache(cacheName); + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/MaintenanceTaskConfiguration.java b/src/main/java/jenkins/plugins/git/maintenance/MaintenanceTaskConfiguration.java new file mode 100644 index 0000000000..573a0cbaf3 --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/MaintenanceTaskConfiguration.java @@ -0,0 +1,184 @@ +package jenkins.plugins.git.maintenance; + +import antlr.ANTLRException; +import com.google.common.collect.ImmutableList; +import edu.umd.cs.findbugs.annotations.NonNull; +import hudson.Extension; +import hudson.Launcher; +import hudson.model.TaskListener; +import hudson.scheduler.CronTab; +import hudson.util.StreamTaskListener; +import jenkins.model.GlobalConfiguration; +import org.apache.commons.lang3.StringUtils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * MaintenanceTaskConfiguration is responsible for creating tasks,fetching tasks, storing the configuration in Jenkins. It extends the + * {@link GlobalConfiguration} class. + * @since TODO + * + * @author Hrushikesh Rao + */ +@Extension +public class MaintenanceTaskConfiguration extends GlobalConfiguration { + + /** + * It is the core data structure which stores all maintenance tasks. + * A map ensure the efficient data access. The {@link TaskType} is key and {@link Task} is value. + * + */ + private Map maintenanceTasks; + private boolean isGitMaintenanceRunning; + + private static final Logger LOGGER = Logger.getLogger(MaintenanceTaskConfiguration.class.getName()); + + /** + * The constructor checks if a maintenanceTaskConfiguration.xml file is present on Jenkins. + * This xml file contains maintenance data configured by the administrator. + * If file is present, the data is loaded from the file. The file would be missing if administrators never configured maintenance task on Jenkins. + * If file is missing a data structure is created by calling {@link MaintenanceTaskConfiguration#configureMaintenanceTasks()}. + * + */ + public MaintenanceTaskConfiguration(){ + + LOGGER.log(Level.FINE,"Loading git-maintenance configuration if present on jenkins controller."); + load(); + if(maintenanceTasks == null) { + LOGGER.log(Level.FINE,"Git maintenance configuration not present on Jenkins, creating a default configuration"); + configureMaintenanceTasks(); + isGitMaintenanceRunning = false; + }else{ + LOGGER.log(Level.FINE,"Loaded git maintenance configuration successfully."); + } + } + + /** + * Initializes a data structure if configured for first time. The data structure is used to store and fetch maintenance configuration. + */ + private void configureMaintenanceTasks(){ + // check git version and based on git version, add the maintenance tasks to the list + // Can add default cron syntax for maintenance tasks. + maintenanceTasks = new LinkedHashMap<>(); + + maintenanceTasks.put(TaskType.COMMIT_GRAPH,new Task(TaskType.COMMIT_GRAPH)); + maintenanceTasks.put(TaskType.PREFETCH,new Task(TaskType.PREFETCH)); + maintenanceTasks.put(TaskType.GC,new Task(TaskType.GC)); + maintenanceTasks.put(TaskType.LOOSE_OBJECTS,new Task(TaskType.LOOSE_OBJECTS)); + maintenanceTasks.put(TaskType.INCREMENTAL_REPACK,new Task(TaskType.INCREMENTAL_REPACK)); + } + + /** + * Gets a copy of list of Maintenance Tasks. + * + * @return List of {@link Task}. + */ + public List getMaintenanceTasks(){ + List maintenanceTasks = new ArrayList<>(); + for(Map.Entry entry : this.maintenanceTasks.entrySet()){ + maintenanceTasks.add(entry.getValue()); + } + return ImmutableList.copyOf(maintenanceTasks); + } + + /** + * Set the cron Syntax of a maintenance task. + * + * @param taskType type of maintenance task. + * @param cronSyntax cron syntax corresponding to task. + */ + public void setCronSyntax(TaskType taskType, String cronSyntax){ + Task updatedTask = maintenanceTasks.get(taskType); + updatedTask.setCronSyntax(cronSyntax); + maintenanceTasks.put(taskType,updatedTask); + LOGGER.log(Level.FINE,"Assigned " + cronSyntax + " to " + taskType.getTaskName()); + } + + /** + * Returns the status of git maintenance i.e. is it configured or not. + * + * @return A boolean if git maintenance is configured globally. + */ + public boolean getIsGitMaintenanceRunning(){ + return isGitMaintenanceRunning; + } + + /** + * Set the execution status of git maintenance globally. If false, the git maintenance is not executed on any cache. + * + * @param executionStatus a boolean to set the global git maintenance. + */ + public void setIsGitMaintenanceRunning(boolean executionStatus){isGitMaintenanceRunning = executionStatus;} + + /** + * Maintenance task state can be changed by toggling the isConfigured boolean variable present in {@link Task} class. + * If isConfigured is true, the maintenance task is executed when corresponding cronSyntax is valid else task is not executed at all. + * + * @param taskType The type of maintenance task. + * @param isConfigured The state of execution of maintenance Task. + */ + public void setIsTaskConfigured(TaskType taskType, boolean isConfigured){ + Task task = maintenanceTasks.get(taskType); + task.setIsTaskConfigured(isConfigured); + LOGGER.log(Level.FINE,taskType.getTaskName() + " execution status: " + isConfigured); + } + + /** + * Validates the input cron syntax. + * + * @param cron Cron syntax as String. + * @return Empty string if no error, else a msg describing error in cron syntax. + * @throws ANTLRException during incorrect cron input. + */ + public static String checkSanity(@NonNull String cron) throws ANTLRException { + try { + CronTab cronTab = new CronTab(cron.trim()); + String msg = cronTab.checkSanity(); + return msg; + }catch(ANTLRException e){ + if(cron.contains("**")) + throw new ANTLRException("You appear to be missing whitespace between * and *."); + throw new ANTLRException(String.format("Invalid input: \"%s\": %s", cron, e), e); + } + } + + /** + * Returns the git version used for maintenance. + * + * @return the git version used for maintenance. + */ + static List getGitVersion(){ + + final TaskListener procListener = StreamTaskListener.fromStderr(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + new Launcher.LocalLauncher(procListener).launch().cmds("git", "--version").stdout(out).join(); + } catch (IOException | InterruptedException ex) { + LOGGER.log(Level.WARNING, "Exception checking git version " + ex); + } + String versionOutput = ""; + try { + versionOutput = out.toString(StandardCharsets.UTF_8.toString()).trim(); + } catch (UnsupportedEncodingException ue) { + LOGGER.log(Level.WARNING, "Unsupported encoding checking git version", ue); + } + final String[] fields = versionOutput.split(" ")[2].replaceAll("msysgit.", "").replaceAll("windows.", "").split("\\."); + + // Eg: [2, 31, 4] + // 0th index is Major Version. + // 1st index is Minor Version. + // 2nd index is Patch Version. + return Arrays.stream(fields).map(Integer::parseInt).collect(Collectors.toList()); + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/MaintenanceUI.java b/src/main/java/jenkins/plugins/git/maintenance/MaintenanceUI.java new file mode 100644 index 0000000000..d258ede50e --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/MaintenanceUI.java @@ -0,0 +1,251 @@ +package jenkins.plugins.git.maintenance; + +import antlr.ANTLRException; +import edu.umd.cs.findbugs.annotations.NonNull; +import hudson.Extension; +import hudson.model.ManagementLink; +import hudson.model.PeriodicWork; +import hudson.security.Permission; +import hudson.util.FormValidation; +import jenkins.model.GlobalConfiguration; +import jenkins.model.Jenkins; +import jenkins.plugins.git.maintenance.Logs.CacheRecord; +import jenkins.plugins.git.maintenance.Logs.XmlSerialize; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; +import org.kohsuke.stapler.QueryParameter; +import org.kohsuke.stapler.StaplerRequest; +import org.kohsuke.stapler.StaplerResponse; +import org.kohsuke.stapler.bind.JavaScriptMethod; +import org.kohsuke.stapler.interceptor.RequirePOST; +import org.kohsuke.stapler.verb.POST; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Extension +public class MaintenanceUI extends ManagementLink { + + JSONObject notification = new JSONObject(); + String OK = "OK"; + String ERROR = "ERROR"; + + private static final Logger LOGGER = Logger.getLogger(MaintenanceUI.class.getName()); + + @Override + public String getIconFileName() { + return jenkins.model.Jenkins.RESOURCE_PATH + "/plugin/git/icons/git-maintenance.svg"; + } + + @Override + public String getDisplayName() { + return "Git Maintenance"; + } + + @Override + public String getUrlName() { + return "maintenance"; + } + + @Override + public String getDescription() { + return "Maintain your repositories to improve git performance."; + } + + public @NonNull String getCategoryName() { + return "CONFIGURATION"; + } + + @RequirePOST + @Restricted(NoExternalUse.class) + public void doSave(StaplerRequest req, StaplerResponse res) throws IOException, ServletException { + if (!Jenkins.get().hasPermission(Jenkins.ADMINISTER)) { + LOGGER.log(Level.WARNING,"User doesn't have the required permission to access git-maintenance."); + res.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + + JSONObject formData = req.getSubmittedForm(); + MaintenanceTaskConfiguration config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + + if(config != null) { + for (TaskType taskType : TaskType.values()) { + JSONObject maintenanceData = formData.getJSONObject(taskType.toString()); + String cronSyntax = maintenanceData.getString("cronSyntax"); + boolean isApplied = maintenanceData.getBoolean("isApplied"); + + config.setCronSyntax(taskType, cronSyntax); + config.setIsTaskConfigured(taskType, isApplied); + } + config.save(); + LOGGER.log(Level.FINE, "Maintenance configuration data stored successfully on Jenkins."); + setNotification("Data saved on Jenkins.",OK); + res.sendRedirect(""); + return; + } + LOGGER.log(Level.WARNING,"Couldn't load Global git maintenance configuration. Internal Error."); + setNotification("Internal Error! Data not saved.",ERROR); + } + + @RequirePOST + @Restricted(NoExternalUse.class) + public void doExecuteMaintenanceTask(StaplerRequest req, StaplerResponse res) throws IOException { + if (!Jenkins.get().hasPermission(Jenkins.ADMINISTER)) { + LOGGER.log(Level.WARNING,"User doesn't have the required permission to access git-maintenance"); + res.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + + MaintenanceTaskConfiguration config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + if(config != null) { + + // Todo + // schedule maintenance tasks only if all cron syntax are valid. + // else can't schedule maintenance tasks. + + boolean updatedGitMaintenanceExecutionStatus = true; + config.setIsGitMaintenanceRunning(updatedGitMaintenanceExecutionStatus); + config.save(); + LOGGER.log(Level.FINE, "Git Maintenance tasks are scheduled for execution."); + setNotification("Scheduled Maintenance Tasks.",OK); + }else{ + setNotification("Internal Error! Tasks not scheduled.",ERROR); + } + + res.sendRedirect(""); + } + + @RequirePOST + @Restricted(NoExternalUse.class) + public void doTerminateMaintenanceTask(StaplerRequest req, StaplerResponse res) throws IOException { + if (!Jenkins.get().hasPermission(Jenkins.ADMINISTER)) { + LOGGER.log(Level.WARNING,"User doesn't have the required permission to access git-maintenance"); + res.sendError(HttpServletResponse.SC_FORBIDDEN); + return; + } + + MaintenanceTaskConfiguration config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + if(config != null) { + boolean updatedGitMaintenanceExecutionStatus = false; + config.setIsGitMaintenanceRunning(updatedGitMaintenanceExecutionStatus); + config.save(); + + Cron cron = PeriodicWork.all().get(Cron.class); + if (cron != null) { + cron.terminateMaintenanceTaskExecution(); + cron.cancel(); + LOGGER.log(Level.FINE, "Terminated scheduling of Git Maintenance tasks."); + setNotification("Terminated Maintenance Tasks.",OK); + } else { + LOGGER.log(Level.WARNING, "Couldn't Terminate Maintenance Task. Internal Error."); + setNotification("Internal Error! Couldn't Terminate Tasks.",ERROR); + } + }else{ + setNotification("Internal Error! Couldn't Terminate Tasks.",ERROR); + } + res.sendRedirect(""); + } + + @POST + @Restricted(NoExternalUse.class) + public FormValidation doCheckCronSyntax(@QueryParameter String cronSyntax) throws ANTLRException { + try { + Jenkins.get().checkPermission(Jenkins.ADMINISTER); + if (cronSyntax.isEmpty()) + return FormValidation.ok(); + + String msg = MaintenanceTaskConfiguration.checkSanity(cronSyntax); + + if (msg != null) { + return FormValidation.error(msg); + } + return FormValidation.ok(); + }catch(ANTLRException e){ + return FormValidation.error(e.getMessage()); + } + } + + public List getMaintenanceTasks(){ + // Can check if git version doesn't support a maintenance task and remove that maintenance task from the UI. + MaintenanceTaskConfiguration config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + if(config != null) + return config.getMaintenanceTasks(); + LOGGER.log(Level.WARNING,"Couldn't load Global git maintenance configuration. Internal Error."); + return new ArrayList<>(); + } + + public boolean getIsGitMaintenanceRunning(){ + MaintenanceTaskConfiguration config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + if(config != null) + return config.getIsGitMaintenanceRunning(); + LOGGER.log(Level.WARNING,"Couldn't load Global git maintenance configuration. Internal Error."); + return false; + } + + public String getGitVersion(){ + List gitVersion = MaintenanceTaskConfiguration.getGitVersion(); + return gitVersion.get(0) + "." + gitVersion.get(1) + "." + gitVersion.get(2); + } + + public String updateGitVersionHelperText(){ + List gitVersion = MaintenanceTaskConfiguration.getGitVersion(); + int gitMajor = gitVersion.get(0); + int gitMinor = gitVersion.get(1); + + if((gitMajor == 2 && gitMinor >= 30) || (gitMajor > 2)) + return ""; + + return "Use git version >= 2.30 to get full benefits of git maintenance."; + } + + @NonNull + @Override + public Permission getRequiredPermission() { + return Jenkins.ADMINISTER; + } + + @JavaScriptMethod + public void setNotification(String notification, String type){ + this.notification.put("msg",notification); + this.notification.put("type",type); + } + + @JavaScriptMethod + public JSONObject getNotification(){ + // creating a copy... + return JSONObject.fromObject(notification.toString()); + } + + public List getMaintenanceRecords(){ + return new XmlSerialize().getMaintenanceRecords(); + } + + @JavaScriptMethod + public JSONObject getRecordsForSingleCache(String cacheName){ + List allMaintenanceRecordsOfCache = new XmlSerialize().getAllMaintenanceRecordsForSingleCache(cacheName); + JSONObject jsonObject = new JSONObject(); + JSONArray jsonArray = new JSONArray(); + + // Converting List of records to Json. + for(CacheRecord cacheRecord : allMaintenanceRecordsOfCache){ + JSONObject cacheObject = new JSONObject(); + cacheObject.put("maintenanceType",cacheRecord.getMaintenanceType()); + cacheObject.put("timeOfExecution",cacheRecord.getTimeOfExecution()); + cacheObject.put("executionStatus",cacheRecord.getExecutionStatus()); + cacheObject.put("executionDuration",cacheRecord.getExecutionDuration()); + cacheObject.put("repoSize",cacheRecord.getRepoSize()); + jsonArray.add(cacheObject); + } + + jsonObject.put("maintenanceData",jsonArray); + return jsonObject; + } + +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/Task.java b/src/main/java/jenkins/plugins/git/maintenance/Task.java new file mode 100644 index 0000000000..389c030c07 --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/Task.java @@ -0,0 +1,88 @@ +package jenkins.plugins.git.maintenance; + +/** + * POJO to store configuration for maintenance tasks. + * @since TODO + * + * @author Hrushikesh Rao + */ +public class Task { + + private TaskType task; + private String cronSyntax; + private boolean isConfigured; + + /** + * Initialize a maintenance task object. + * + * @param task {@link TaskType}. + */ + public Task(TaskType task){ + // Can add default cron syntax recommended by git documentation + this.task = task; + } + + /** + * A convenience constructor that copies the Task object. + * + * @param copyTask task object. + * + */ + public Task(Task copyTask){ + // Used for copying the task; + this(copyTask.getTaskType()); + setCronSyntax(copyTask.getCronSyntax()); + setIsTaskConfigured(copyTask.getIsTaskConfigured()); + } + + /** + * Gets the TaskType enum. + * + * @return TaskType {@link TaskType}. + */ + public TaskType getTaskType(){ + return this.task; + } + + /** + * Gets the name of the maintenance task. + * + * @return maintenance task name. + */ + public String getTaskName(){ + return this.task.getTaskName(); + } + + /** + * + * @return cron syntax configured for a maintenance task. + */ + public String getCronSyntax(){ return this.cronSyntax; } + + /** + * Toggle the state of execution of maintenance task. + * + * @param isConfigured if true, corresponding task is configured and executed using cron syntax. + */ + public void setIsTaskConfigured(boolean isConfigured){ + this.isConfigured = isConfigured; + } + + /** + * + * @return A boolean to check maintenance task is configured by administrator. + */ + public boolean getIsTaskConfigured(){ + return this.isConfigured; + } + + /** + * Configure the Cron Syntax + * + * @param cronSyntax cron syntax as String + * + */ + public void setCronSyntax(String cronSyntax){ + this.cronSyntax = cronSyntax; + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/TaskExecutor.java b/src/main/java/jenkins/plugins/git/maintenance/TaskExecutor.java new file mode 100644 index 0000000000..5124db6b1d --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/TaskExecutor.java @@ -0,0 +1,237 @@ +package jenkins.plugins.git.maintenance; + +import hudson.FilePath; +import hudson.model.TaskListener; +import hudson.plugins.git.GitTool; +import hudson.plugins.git.util.GitUtils; +import hudson.util.LogTaskListener; +import jenkins.model.Jenkins; +import jenkins.plugins.git.maintenance.Logs.CacheRecord; +import jenkins.plugins.git.maintenance.Logs.XmlSerialize; +import org.apache.commons.io.FileUtils; +import org.jenkinsci.plugins.gitclient.CliGitAPIImpl; +import org.jenkinsci.plugins.gitclient.Git; +import org.jenkinsci.plugins.gitclient.GitClient; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * TaskExecutor executes the maintenance tasks on all Caches on Jenkins controller. A lock is added to each cache before running maintenance on it. + * It is an independent thread. If a cache is already locked, it skips the maintenance task. {@link GitClient} manages the execution of maintenance. + * + * @author Hrushikesh Rao + */ +public class TaskExecutor implements Runnable { + + /** + * Boolean to toggle the state of execution of thread. + */ + private volatile boolean isThreadAlive; + + /** + * Type of maintenance task being executed on caches. + */ + Task maintenanceTask; + + /** + * List of caches present on Jenkins controller. + */ + private List caches; + + XmlSerialize xmlSerialize; + + private static final Logger LOGGER = Logger.getLogger(TaskExecutor.class.getName()); + + /** + * Initializes the thread to execute a maintenance task. + * + * @param maintenanceTask Type of maintenance task required for execution. + */ + public TaskExecutor(Task maintenanceTask){ + this.maintenanceTask = new Task(maintenanceTask); + caches = getCaches(); + isThreadAlive = true; + xmlSerialize = new XmlSerialize(); + LOGGER.log(Level.FINE,"New Thread created to execute " + maintenanceTask.getTaskName()); + } + + /** + * Executes the maintenance task by iterating through the all caches on Jenkins controller. + */ + @Override + public void run() { + + LOGGER.log(Level.INFO,"Executing maintenance task " + maintenanceTask.getTaskName() + " on git caches."); + try { + for (GitMaintenanceSCM.Cache cache : caches) { + // For now adding lock to all kinds of maintenance tasks. Need to study on which task needs a lock and which doesn't. + Lock lock = cache.getLock(); + File cacheFile = cache.getCacheFile(); + executeMaintenanceTask(cacheFile,lock); + } + + LOGGER.log(Level.INFO,maintenanceTask.getTaskName() + " has been executed successfully."); + }catch (InterruptedException e){ + LOGGER.log(Level.WARNING,"Interrupted Exception. Msg: " + e.getMessage()); + } + + } + + /** + * Executes maintenance task on a single cache. The cache is first locked and then undergoes git maintenance. + * + * @param cacheFile File of a single cache. + * @param lock Lock for a single cache. + * @throws InterruptedException When GitClient is interrupted during maintenance execution. + */ + void executeMaintenanceTask(File cacheFile,Lock lock) throws InterruptedException{ + + TaskType taskType = maintenanceTask.getTaskType(); + long executionDuration = 0; + boolean executionStatus = false; + GitClient gitClient; + // If lock is not available on the cache, skip maintenance on this cache. + if (isThreadAlive && lock.tryLock()) { + + LOGGER.log(Level.FINE, "Cache " + cacheFile.getName() + " locked."); + + try { + gitClient = getGitClient(cacheFile); + if (gitClient == null) + throw new InterruptedException("Git Client couldn't be instantiated"); + + executionDuration -= System.currentTimeMillis(); + + executionStatus = executeGitMaintenance(gitClient,taskType); + + executionDuration += System.currentTimeMillis(); + } catch (InterruptedException e) { + LOGGER.log(Level.FINE, "Couldn't run " + taskType.getTaskName() + ".Msg: " + e.getMessage()); + } finally { + lock.unlock(); + LOGGER.log(Level.FINE, "Cache " + cacheFile.getName() + " unlocked."); + } + + } else { + if(!isThreadAlive) + throw new InterruptedException("Maintenance thread has been interrupted. Terminating..."); + else + LOGGER.log(Level.FINE,"Cache is already locked. Can't run maintenance on cache " + cacheFile.getName()); + } + + xmlSerialize.addMaintenanceRecord(createRecord(cacheFile,taskType,executionStatus,executionDuration)); // Stores the record in jenkins. + } + + /** + * Executes git maintenance on single cache.{@link TaskType} defines the type of maintenance task. + * + * @param gitClient {@link GitClient} on a single cache. + * @param taskType Type of maintenance task. + * @return Boolean if maintenance has been executed or not. + * @throws InterruptedException When GitClient is interrupted during maintenance execution. + */ + boolean executeGitMaintenance(GitClient gitClient,TaskType taskType) throws InterruptedException { + boolean isExecuted = false; + switch (taskType){ + case GC: + isExecuted = gitClient.maintenance("gc"); + break; + case COMMIT_GRAPH: + isExecuted = gitClient.maintenance("commit-graph"); + break; + case PREFETCH: + isExecuted = gitClient.maintenance("prefetch"); + break; + case INCREMENTAL_REPACK: + isExecuted = gitClient.maintenance("incremental-repack"); + break; + case LOOSE_OBJECTS: + isExecuted = gitClient.maintenance("loose-objects"); + break; + default: + LOGGER.log(Level.WARNING,"Invalid maintenance task."); + terminateThread(); + } + return isExecuted; + } + + /** + * Returns a list of caches present on Jenkins controller. See {@link GitMaintenanceSCM#getCaches()} + * + * @return List of caches on Jenkins controller. + */ + List getCaches(){ + List caches = GitMaintenanceSCM.getCaches(); + LOGGER.log(Level.FINE,"Fetched all caches present on Jenkins Controller."); + return caches; + } + + /** + * Returns {@link GitClient} on a single cache. + * + * @param file File object of a single cache. + * @return GitClient on single cache. + */ + GitClient getGitClient(File file){ + try { + TaskListener listener = new LogTaskListener(LOGGER, Level.FINE); + final Jenkins jenkins = Jenkins.getInstanceOrNull(); + GitTool gitTool = GitUtils.resolveGitTool(null, jenkins, null, listener); + if (gitTool == null) { + LOGGER.log(Level.WARNING, "No GitTool found while running " + maintenanceTask.getTaskName()); + return null; + } + + String gitExe = gitTool.getGitExe(); + if (file != null) { + FilePath workspace = new FilePath(file); + Git git = Git.with(listener, null).in(workspace).using(gitExe); + + GitClient gitClient = git.getClient(); + if (gitClient instanceof CliGitAPIImpl) + return gitClient; + LOGGER.log(Level.WARNING, "JGit requested, but does not execute maintenance tasks"); + } else { + LOGGER.log(Level.WARNING, "Cli Git will not execute maintenance tasks due to null file arg"); + } + + }catch (InterruptedException | IOException e ){ + LOGGER.log(Level.WARNING,"Git Client couldn't be initialized."); + } + return null; + } + + /** + * Terminates this thread and stops further execution of maintenance tasks. + */ + public void terminateThread(){ + isThreadAlive = false; + } + + /** + * Returns a Record ({@link CacheRecord}) which contains the result of maintenance data on a single cache. + * + * @param cacheFile File object of a single cache. + * @param taskType Type of maintenance task. + * @param executionStatus Whether maintenance task has been executed successfully on a cache. + * @param executionDuration Amount of time taken to execute maintenance task in milliseconds. + * @return {@link CacheRecord} of a single cache. + */ + CacheRecord createRecord(File cacheFile, TaskType taskType,boolean executionStatus,long executionDuration){ + + CacheRecord cacheRecord = new CacheRecord(cacheFile.getName(),taskType.getTaskName()); + long repoSizeInBytes = FileUtils.sizeOfDirectory(cacheFile); + cacheRecord.setRepoSize(repoSizeInBytes); + cacheRecord.setExecutionStatus(executionStatus); + if(!executionStatus) + cacheRecord.setExecutionDuration(-1); + else cacheRecord.setExecutionDuration(executionDuration); + cacheRecord.setTimeOfExecution(System.currentTimeMillis()/1000); // Store the unix timestamp + return cacheRecord; + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/TaskScheduler.java b/src/main/java/jenkins/plugins/git/maintenance/TaskScheduler.java new file mode 100644 index 0000000000..cf0999ee23 --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/TaskScheduler.java @@ -0,0 +1,182 @@ +package jenkins.plugins.git.maintenance; + +import antlr.ANTLRException; +import hudson.scheduler.CronTab; +import hudson.scheduler.CronTabList; +import hudson.scheduler.Hash; +import jenkins.model.GlobalConfiguration; + +import java.util.Calendar; +import java.util.Collections; +import java.util.GregorianCalendar; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * TaskScheduler is responsible for scheduling maintenance tasks. It validates if a task is configured and verifies cron syntax before scheduling. + * TaskScheduler acts as a producer by adding the appropriate task into a maintenance queue. + * + * @author Hrushikesh Rao + */ +public class TaskScheduler { + + private MaintenanceTaskConfiguration config; + + /** + * Stores the order of execution of maintenance tasks. + */ + private List maintenanceQueue; + + /** + * {@link TaskExecutor} executes the maintenance tasks on all the caches. + */ + private Thread taskExecutor; + private TaskExecutor taskExecutorRunnable; + + private static final Logger LOGGER = Logger.getLogger(TaskScheduler.class.getName()); + private static final Random random = new Random(); + + /** + * Loads the maintenance configuration configured by the administrator. Also initializes an empty maintenance queue. + */ + public TaskScheduler(){ + this.config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + this.maintenanceQueue = new LinkedList<>(); + LOGGER.log(Level.FINE,"TaskScheduler class Initialized."); + } + + /** + * Schedules maintenance tasks on caches. + */ + void scheduleTasks() { + if(config != null) { + if (!isGitMaintenanceTaskRunning(config)) { + // Logs ever 1 min. Need to check performance impact. + LOGGER.log(Level.FINER, "Maintenance Task execution not configured in UI."); + return; + } + + List maintenanceTasks = config.getMaintenanceTasks(); + addTasksToQueue(maintenanceTasks); + // Option of Using the same thread for executing more maintenance task, or create a new thread the next minute and execute the maintenance task. + createTaskExecutorThread(); + }else{ + LOGGER.log(Level.FINE,"Couldn't load Global git maintenance configuration. Internal Error."); + } + + } + + /** + * Checks for duplication of maintenance task in the queue. + * + * @param task type of maintenance task ({@link Task}). + * @return a boolean to see if a task is already present in maintenance queue. + */ + boolean checkIsTaskInQueue(Task task){ + boolean isTaskInQueue = maintenanceQueue.stream().anyMatch(queuedTask -> queuedTask.getTaskType().equals(task.getTaskType())); + if(isTaskInQueue){ + LOGGER.log(Level.FINE,task.getTaskName() + " is already present in maintenance queue."); + } + return isTaskInQueue; + } + + /** + * A new Thread {@link TaskExecutor} is created which executes the maintenance task on caches. + */ + void createTaskExecutorThread(){ + // Create a new thread and execute the tasks present in the queue; + if(!maintenanceQueue.isEmpty() && (taskExecutor == null || !taskExecutor.isAlive())) { + Task currentTask = maintenanceQueue.remove(0); + taskExecutorRunnable = new TaskExecutor(currentTask); + taskExecutor = new Thread(taskExecutorRunnable, "maintenance-task-executor"); + taskExecutor.start(); + LOGGER.log(Level.INFO,"Thread [" + taskExecutor.getName() +"] created to execute " + currentTask.getTaskName() + " task."); + } + } + + /** + * Iterates through all the maintenance tasks and adds the task to queue if valid. + * + * @param maintenanceTasks List of maintenance tasks {@link Task} + */ + void addTasksToQueue(List maintenanceTasks){ + + boolean isTaskExecutable; + for(Task task : maintenanceTasks){ + try { + if(!task.getIsTaskConfigured() || checkIsTaskInQueue(task)) + continue; + + CronTabList cronTabList = getCronTabList(task.getCronSyntax()); + isTaskExecutable = checkIsTaskExecutable(cronTabList); + if(isTaskExecutable){ + maintenanceQueue.add(task); + LOGGER.log(Level.INFO,task.getTaskName() + " added to maintenance queue."); + } + }catch (ANTLRException e){ + // Logged every minute. Need to check performance. + LOGGER.log(Level.WARNING,"Invalid cron syntax:[ "+ task.getTaskName() + " ]" + ",msg: " + e.getMessage()); + } + } + } + + /** + * Checks if global git maintenance is configured. + * @param config {@link MaintenanceTaskConfiguration}. + * @return Global git maintenance is configured or not. + */ + boolean isGitMaintenanceTaskRunning(MaintenanceTaskConfiguration config){ + return config.getIsGitMaintenanceRunning(); + } + + /** + * Returns CronTabList for a cronSyntax. + * + * @param cronSyntax cron syntax for maintenance task. + * @return {@link CronTabList} + * @throws ANTLRException + */ + CronTabList getCronTabList(String cronSyntax) throws ANTLRException { + // Random number between 0 & 100000 + String seed = String.valueOf((random.nextInt(100000))); + CronTab cronTab = new CronTab(cronSyntax.trim(), Hash.from(seed)); + return new CronTabList(Collections.singletonList(cronTab)); + } + + /** + * Checks if the cron syntax matches the current time. Returns true if valid. + * + * @param cronTabList {@link CronTabList} + * @return maintenance task should be scheduled or not. + */ + boolean checkIsTaskExecutable(CronTabList cronTabList){ + boolean isTaskExecutable = false; + + Calendar cal = new GregorianCalendar(); + isTaskExecutable = cronTabList.check(cal); + // Further validation such as not schedule a task every minute etc. can be added here. + + return isTaskExecutable; + } + + /** + * Terminates the {@link TaskExecutor} thread to stop executing maintenance on caches. Also empties the maintenance queue. + */ + void terminateMaintenanceTaskExecution(){ + this.maintenanceQueue = new LinkedList<>(); + if(taskExecutor != null && taskExecutor.isAlive()) + taskExecutorRunnable.terminateThread(); + + LOGGER.log(Level.INFO,"Terminated Execution of maintenance tasks"); + } + + List getMaintenanceQueue(){ + return maintenanceQueue; + } + Thread getTaskExecutor(){ + return taskExecutor; + } +} diff --git a/src/main/java/jenkins/plugins/git/maintenance/TaskType.java b/src/main/java/jenkins/plugins/git/maintenance/TaskType.java new file mode 100644 index 0000000000..1074a8f9ff --- /dev/null +++ b/src/main/java/jenkins/plugins/git/maintenance/TaskType.java @@ -0,0 +1,39 @@ +package jenkins.plugins.git.maintenance; + +/** + * TaskType describes the type of maintenance task. There are 5 types of maintenance tasks. They are- + *
    + *
  • Prefetch
  • + *
  • Garbage Collection
  • + *
  • Commit Graph
  • + *
  • Loose Objects
  • + *
  • Incremental Repack
  • + *
+ * + * @author Hrushikesh Rao + */ +public enum TaskType { + GC("Garbage Collection"), + PREFETCH("Prefetch"), + COMMIT_GRAPH("Commit Graph"), + LOOSE_OBJECTS("Loose Objects"), + INCREMENTAL_REPACK("Incremental Repack"); + + String taskName; + + /** + * + * @param taskName Assign a name for maintenance task. + */ + TaskType(String taskName){ + this.taskName = taskName; + } + + /** + * + * @return name of the maintenance task. + */ + public String getTaskName(){ + return this.taskName; + } +} diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-commit-graph.html b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-commit-graph.html new file mode 100644 index 0000000000..50fe86c862 --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-commit-graph.html @@ -0,0 +1,3 @@ +
+ Help regarding commit graph +
\ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-gc.html b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-gc.html new file mode 100644 index 0000000000..b0ad57c725 --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-gc.html @@ -0,0 +1,3 @@ +
+ Help regarding gc +
diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-incremental-repack.html b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-incremental-repack.html new file mode 100644 index 0000000000..f4feaf18a4 --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-incremental-repack.html @@ -0,0 +1,3 @@ +
+ Help regarding incremental repack +
\ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-loose-objects.html b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-loose-objects.html new file mode 100644 index 0000000000..71d2dc5714 --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-loose-objects.html @@ -0,0 +1,3 @@ +
+ Help regarding loose-objects +
\ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-prefetch.html b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-prefetch.html new file mode 100644 index 0000000000..1e18161577 --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/help-prefetch.html @@ -0,0 +1,3 @@ +
+ Help regarding prefetch +
\ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/index.jelly b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/index.jelly new file mode 100644 index 0000000000..601436029a --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/index.jelly @@ -0,0 +1,64 @@ + + + + + +
+

Maintenance Tasks

+

Git Version: ${it.getGitVersion()}

+

${it.updateGitVersionHelperText()}

+

Enter cron syntax to execute maintenance tasks periodically

+ + +
+ + + + +
+
+ +
+ + + + + + + + + + + +
+ + + +
+ +
+

Maintenance Records

+ + +
+ +
+
+
diff --git a/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/table.jelly b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/table.jelly new file mode 100644 index 0000000000..c232f8c0dc --- /dev/null +++ b/src/main/resources/jenkins/plugins/git/maintenance/MaintenanceUI/table.jelly @@ -0,0 +1,153 @@ + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#${%Cache Name}${%Cache Size} (MB)${%Task}${%Time}${%Status}${%Execution Time} (mmm:ss:SSS)Expand
+ ${loop.count} + + ${record.getRepoName()} + + + ${record.getRepoSize()/1000000} + + ${record.getMaintenanceType()} + + ${record.getTimeOfExecution()} + + + + + + + + + ${record.getExecutionDuration()} + + Expand +
#${%Cache Name}${%Cache Size} (MB)${%Task}${%Time}${%Status}${%Execution Time} (mmm:ss:SSS)Expand
+
+
+ + + +
diff --git a/src/main/webapp/icons/git-maintenance.svg b/src/main/webapp/icons/git-maintenance.svg new file mode 100644 index 0000000000..66ab7a3350 --- /dev/null +++ b/src/main/webapp/icons/git-maintenance.svg @@ -0,0 +1,51 @@ + + + + +Created by potrace 1.15, written by Peter Selinger 2001-2017 + + + + + diff --git a/src/test/java/hudson/plugins/git/extensions/impl/CloneOptionHonorRefSpecTest.java b/src/test/java/hudson/plugins/git/extensions/impl/CloneOptionHonorRefSpecTest.java index d963bd4923..834eeddd74 100644 --- a/src/test/java/hudson/plugins/git/extensions/impl/CloneOptionHonorRefSpecTest.java +++ b/src/test/java/hudson/plugins/git/extensions/impl/CloneOptionHonorRefSpecTest.java @@ -105,7 +105,6 @@ public void testRefSpecWithExpandedVariables() throws Exception { if (refSpecExpectedValue == null || refSpecExpectedValue.isEmpty()) { /* Test does not support an empty or null expected value. Skip the test if the expected value is empty or null */ - System.out.println("*** testRefSpecWithExpandedVariables empty expected value for '" + refSpecName + "' ***"); return; } // Create initial commit diff --git a/src/test/java/hudson/plugins/git/util/GitUtilsTest.java b/src/test/java/hudson/plugins/git/util/GitUtilsTest.java index 5e793eeb13..d2c6cbbfce 100644 --- a/src/test/java/hudson/plugins/git/util/GitUtilsTest.java +++ b/src/test/java/hudson/plugins/git/util/GitUtilsTest.java @@ -98,7 +98,7 @@ public class GitUtilsTest { private GitUtils gitUtils; private static GitClient gitClient; - private static final Random RANDOM = new Random(); + private static final Random random = new Random(); @BeforeClass public static void createSampleOriginRepo() throws Exception { @@ -121,14 +121,14 @@ public static void createSampleOriginRepo() throws Exception { priorBranchSpecList.add(new BranchSpec(OLDER_BRANCH_NAME)); originRepo.git("checkout", "master"); - originRepo.write(fileName, "This is the " + HEAD_TAG_NAME_0 + " README file " + RANDOM.nextInt()); + originRepo.write(fileName, "This is the " + HEAD_TAG_NAME_0 + " README file " + random.nextInt()); originRepo.git("add", fileName); originRepo.git("commit", "-m", "Adding " + fileName + " tagged " + HEAD_TAG_NAME_0, fileName); originRepo.git("tag", HEAD_TAG_NAME_0); headTag0Id = ObjectId.fromString(originRepo.head()); headTag0Revision = new Revision(headTag0Id); - originRepo.write(fileName, "This is the README file " + RANDOM.nextInt()); + originRepo.write(fileName, "This is the README file " + random.nextInt()); originRepo.git("add", fileName); originRepo.git("commit", "-m", "Adding " + fileName, fileName); originRepo.git("tag", HEAD_TAG_NAME_1); diff --git a/src/test/java/jenkins/plugins/git/maintenance/GitMaintenanceSCMTest.java b/src/test/java/jenkins/plugins/git/maintenance/GitMaintenanceSCMTest.java new file mode 100644 index 0000000000..bec193df3e --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/GitMaintenanceSCMTest.java @@ -0,0 +1,74 @@ +package jenkins.plugins.git.maintenance; + +import hudson.plugins.git.AbstractGitRepository; +import hudson.plugins.git.AbstractGitTestCase; +import hudson.plugins.git.BranchSpec; +import hudson.plugins.git.GitSCM; +import hudson.plugins.git.extensions.GitSCMExtension; +import jenkins.model.Jenkins; +import jenkins.plugins.git.AbstractGitSCMSource; +import jenkins.plugins.git.AbstractGitSCMSourceTest; +import jenkins.plugins.git.GitSampleRepoRule; +import jenkins.scm.api.SCMFileSystem; +import org.apache.commons.io.FileUtils; +import org.junit.*; +import org.junit.jupiter.api.BeforeAll; +import org.junit.rules.TemporaryFolder; +import org.jvnet.hudson.test.JenkinsRule; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.configuration.IMockitoConfiguration; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Level; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.*; + +public class GitMaintenanceSCMTest { + + @Rule + public JenkinsRule rule = new JenkinsRule(); + + + @Test + public void testGetCaches() throws Exception{ + + // Cannot test this due to static methods not public, + // AbstractGitSCmSource is in different package + List caches = new ArrayList<>(); + caches.add(new GitMaintenanceSCM.Cache(new File("test.txt"),new ReentrantLock())); + + + mockStatic(GitMaintenanceSCM.class) + .when(()->GitMaintenanceSCM.getCaches()) + .thenReturn(caches); + + assertEquals(caches.size(),GitMaintenanceSCM.getCaches().size()); + + } + + @Test + public void testGetCacheFile(){ + File file = Jenkins.getInstanceOrNull().getRootDir(); + Lock lock = new ReentrantLock(); + GitMaintenanceSCM.Cache cache = new GitMaintenanceSCM.Cache(file,lock); + assertEquals(file.getAbsolutePath(),cache.getCacheFile().getAbsolutePath()); + } + + @Test + public void testGetLock(){ + File file = Jenkins.getInstanceOrNull().getRootDir(); + Lock lock = new ReentrantLock(); + GitMaintenanceSCM.Cache cache = new GitMaintenanceSCM.Cache(file,lock); + assertNotNull(cache.getLock()); + } +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/Logs/CacheRecordTest.java b/src/test/java/jenkins/plugins/git/maintenance/Logs/CacheRecordTest.java new file mode 100644 index 0000000000..0fd27320d9 --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/Logs/CacheRecordTest.java @@ -0,0 +1,149 @@ +package jenkins.plugins.git.maintenance.Logs; + +import io.jenkins.cli.shaded.org.apache.commons.lang.time.DurationFormatUtils; +import jenkins.plugins.git.maintenance.TaskType; +import org.junit.Before; +import org.junit.Test; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class CacheRecordTest { + + CacheRecord record; + + @Before + public void setUp() throws Exception { + record = new CacheRecord("git-plugin", TaskType.GC.getTaskName()); + } + + @Test + public void testGetRepoName(){ + assertEquals("git-plugin",record.getRepoName()); + } + + @Test + public void testSetRepoSize(){ + record.setRepoSize(12343); + assertEquals(12343,record.getRepoSize()); + } + + @Test + public void testSetMaintenanceType(){ + assertEquals(record.getMaintenanceType(),TaskType.GC.getTaskName()); + + record.setMaintenanceType(TaskType.PREFETCH.getTaskName()); + assertEquals(TaskType.PREFETCH.getTaskName(),record.getMaintenanceType()); + } + + @Test + public void testSetTimeOfExecution(){ + long timeOfExecution = 1661552520; + record.setTimeOfExecution(timeOfExecution); + + Date date = new Date(timeOfExecution * 1000L); + SimpleDateFormat sdf = new SimpleDateFormat("dd-MM-yyyy HH:mm"); + assertEquals(sdf.format(date),record.getTimeOfExecution()); + } + + @Test + public void testSetExecutionStatus(){ + record.setExecutionStatus(false); + assertFalse(record.getExecutionStatus()); + + record.setExecutionStatus(true); + assertTrue(record.getExecutionStatus()); + } + + @Test + public void testSetExecutionDuration(){ + long duration = System.currentTimeMillis(); + record.setExecutionDuration(duration); + assertEquals(DurationFormatUtils.formatDuration(duration,"mmm:ss:SSS"),record.getExecutionDuration()); + } + + @Test + public void testInsertMaintenanceData(){ + // Can add other metadata to these caches. + CacheRecord gcCacheRecord = new CacheRecord("git-plugin",TaskType.GC.getTaskName()); + CacheRecord prefetchRecord = new CacheRecord("git-plugin",TaskType.PREFETCH.getTaskName()); + CacheRecord commitGraphRecord = new CacheRecord("git-plugin",TaskType.COMMIT_GRAPH.getTaskName()); + CacheRecord incrementalRepackRecord = new CacheRecord("git-plugin",TaskType.INCREMENTAL_REPACK.getTaskName()); + CacheRecord looseObjectsRecord = new CacheRecord("git-plugin",TaskType.LOOSE_OBJECTS.getTaskName()); + + record.insertMaintenanceData(gcCacheRecord); + record.insertMaintenanceData(prefetchRecord); + record.insertMaintenanceData(commitGraphRecord); + record.insertMaintenanceData(incrementalRepackRecord); + record.insertMaintenanceData(looseObjectsRecord); + + for(Map.Entry> entry : record.maintenanceData.entrySet()){ + assertEquals(1,entry.getValue().size()); + } + } + + @Test + public void getAllMaintenanceRecordsForSingleCache(){ + CacheRecord gcCacheRecord = new CacheRecord("git-plugin",TaskType.GC.getTaskName()); + CacheRecord prefetchRecord = new CacheRecord("git-plugin",TaskType.PREFETCH.getTaskName()); + CacheRecord commitGraphRecord = new CacheRecord("git-plugin",TaskType.COMMIT_GRAPH.getTaskName()); + CacheRecord incrementalRepackRecord = new CacheRecord("git-plugin",TaskType.INCREMENTAL_REPACK.getTaskName()); + CacheRecord looseObjectsRecord = new CacheRecord("git-plugin",TaskType.LOOSE_OBJECTS.getTaskName()); + + // set the TimeofExecution for each cache + gcCacheRecord.setTimeOfExecution(1661552520); + prefetchRecord.setTimeOfExecution(1661553520); + commitGraphRecord.setTimeOfExecution(1661552120); + incrementalRepackRecord.setTimeOfExecution(1661572520); + looseObjectsRecord.setTimeOfExecution(1661512520); + + record.insertMaintenanceData(gcCacheRecord); + record.insertMaintenanceData(prefetchRecord); + record.insertMaintenanceData(commitGraphRecord); + record.insertMaintenanceData(incrementalRepackRecord); + record.insertMaintenanceData(looseObjectsRecord); + + // checking if the data received is in sorted manner or not. + boolean isSorted = true; + + List cacheRecords = record.getAllMaintenanceRecordsForSingleCache(); + + for(int i=1;i cacheRecords.get(i-1).timeOfExecution) { + isSorted = false; + break; + } + } + + assertTrue(isSorted); + } + + @Test + public void copyCacheRecord(){ + long duration = System.currentTimeMillis(); + long timeOfExecution = 1661552520; + + record.setExecutionDuration(duration); + record.setExecutionStatus(true); + record.setTimeOfExecution(timeOfExecution); + record.setRepoSize(12343); + CacheRecord copyCacheRecord = new CacheRecord(record); + + assertEquals(DurationFormatUtils.formatDuration(duration,"mmm:ss:SSS"),copyCacheRecord.getExecutionDuration()); + assertTrue(record.getExecutionStatus()); + assertEquals(TaskType.GC.getTaskName(),copyCacheRecord.getMaintenanceType()); + assertEquals(12343,copyCacheRecord.getRepoSize()); + + Date date = new Date(timeOfExecution * 1000L); + SimpleDateFormat sdf = new SimpleDateFormat("dd-MM-yyyy HH:mm"); + assertEquals(sdf.format(date),copyCacheRecord.getTimeOfExecution()); + } + + + +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/Logs/RecordListTest.java b/src/test/java/jenkins/plugins/git/maintenance/Logs/RecordListTest.java new file mode 100644 index 0000000000..906f7205b6 --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/Logs/RecordListTest.java @@ -0,0 +1,110 @@ +package jenkins.plugins.git.maintenance.Logs; + +import jenkins.plugins.git.maintenance.TaskType; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.MatcherAssert.assertThat; + +public class RecordListTest { + + RecordList recordList; + + String cacheName = "git-plugin"; + @Before + public void setUp() throws Exception { + recordList = new RecordList(); + } + + @Test + public void testGetAllMaintenanceRecords() { + assertThat(recordList.getMaintenanceRecords(),instanceOf(List.class)); + } + + @Test + public void testAddRecord() { + // Testing when adding the first cache record + CacheRecord record = new CacheRecord(cacheName, TaskType.GC.getTaskName()); + record.setExecutionStatus(true); + recordList.addRecord(record); + assertEquals(1,recordList.getMaintenanceRecords().size()); + assertTrue(recordList.getCacheSet().contains(cacheName)); + + // Testing the head of the List. + CacheRecord headRecord = recordList.getMaintenanceRecords().get(0); + assertEquals(TaskType.GC.getTaskName(),headRecord.getMaintenanceType()); + assertEquals(cacheName,headRecord.getRepoName()); + assertTrue(headRecord.getExecutionStatus()); + + + // Testing when adding more records for same cache + CacheRecord gcCacheRecord = new CacheRecord(cacheName,TaskType.GC.getTaskName()); + CacheRecord prefetchRecord = new CacheRecord(cacheName,TaskType.PREFETCH.getTaskName()); + CacheRecord commitGraphRecord = new CacheRecord(cacheName,TaskType.COMMIT_GRAPH.getTaskName()); + CacheRecord incrementalRepackRecord = new CacheRecord(cacheName,TaskType.INCREMENTAL_REPACK.getTaskName()); + CacheRecord looseObjectsRecord = new CacheRecord(cacheName,TaskType.LOOSE_OBJECTS.getTaskName()); + looseObjectsRecord.setExecutionStatus(false); + + recordList.addRecord(gcCacheRecord); + recordList.addRecord(prefetchRecord); + recordList.addRecord(commitGraphRecord); + recordList.addRecord(incrementalRepackRecord); + recordList.addRecord(looseObjectsRecord); + + assertEquals(1,recordList.getMaintenanceRecords().size()); + assertTrue(recordList.getCacheSet().contains(cacheName)); + + // Here the head of the List Cache data will be updated with the latest maintenance data. + headRecord = recordList.getMaintenanceRecords().get(0); + assertEquals(cacheName,headRecord.getRepoName()); + assertEquals(TaskType.LOOSE_OBJECTS.getTaskName(),headRecord.getMaintenanceType()); + assertFalse(headRecord.getExecutionStatus()); + } + + @Test + public void testGetAllMaintenanceRecordsForSingleCache(){ + // Adding record for new Cache + CacheRecord record = new CacheRecord(cacheName, TaskType.GC.getTaskName()); + record.setTimeOfExecution(1661593520); + recordList.addRecord(record); + + CacheRecord gcCacheRecord = new CacheRecord(cacheName,TaskType.GC.getTaskName()); + CacheRecord prefetchRecord = new CacheRecord(cacheName,TaskType.PREFETCH.getTaskName()); + CacheRecord commitGraphRecord = new CacheRecord(cacheName,TaskType.COMMIT_GRAPH.getTaskName()); + CacheRecord incrementalRepackRecord = new CacheRecord(cacheName,TaskType.INCREMENTAL_REPACK.getTaskName()); + CacheRecord looseObjectsRecord = new CacheRecord(cacheName,TaskType.LOOSE_OBJECTS.getTaskName()); // This is the latest cache record + + // set the TimeofExecution for each cache + gcCacheRecord.setTimeOfExecution(1661552520); + prefetchRecord.setTimeOfExecution(1661553520); + commitGraphRecord.setTimeOfExecution(1661552120); + incrementalRepackRecord.setTimeOfExecution(1661572520); + looseObjectsRecord.setTimeOfExecution(1661512520); + + recordList.addRecord(gcCacheRecord); + recordList.addRecord(prefetchRecord); + recordList.addRecord(commitGraphRecord); + recordList.addRecord(incrementalRepackRecord); + recordList.addRecord(looseObjectsRecord); + + List allMaintenanceRecordsForSingleCache = recordList.getAllMaintenanceRecordsForSingleCache(cacheName); + + assertEquals(6,allMaintenanceRecordsForSingleCache.size()); + + boolean isSorted = true; + + for(int i=1;i allMaintenanceRecordsForSingleCache.get(i-1).timeOfExecution){ + isSorted = false; + break; + } + } + + assertTrue(isSorted); + } + +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/MaintenanceTaskConfigurationTest.java b/src/test/java/jenkins/plugins/git/maintenance/MaintenanceTaskConfigurationTest.java new file mode 100644 index 0000000000..93c455ab8b --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/MaintenanceTaskConfigurationTest.java @@ -0,0 +1,99 @@ +package jenkins.plugins.git.maintenance; + +import antlr.ANTLRException; +import org.junit.ClassRule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; + +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.number.OrderingComparison.greaterThan; +import static org.hamcrest.number.OrderingComparison.lessThan; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class MaintenanceTaskConfigurationTest { + + @ClassRule + public static JenkinsRule j = new JenkinsRule(); + private final MaintenanceTaskConfiguration config = new MaintenanceTaskConfiguration(); + + @Test + public void setCronSyntax(){ + String cronSyntax = "* * * 1 *"; + for(TaskType taskType : TaskType.values()){ + config.setCronSyntax(taskType,cronSyntax); + } + + for(TaskType taskType : TaskType.values()){ + List maintenanceTasks = config.getMaintenanceTasks(); + for(Task task : maintenanceTasks){ + if(task.getTaskType().equals(taskType)){ + assertThat(task.getCronSyntax(), is(cronSyntax)); + break; + } + } + } + } + + @Test + public void setIsMaintenanceTaskConfigured(){ + boolean isMaintenanceTaskConfigured = true; + + for(TaskType taskType : TaskType.values()){ + config.setIsTaskConfigured(taskType,isMaintenanceTaskConfigured); + } + + for(TaskType taskType : TaskType.values()){ + List maintenanceTasks = config.getMaintenanceTasks(); + for(Task task : maintenanceTasks){ + if(task.getTaskType().equals(taskType)){ + assertThat(task.getIsTaskConfigured(), is(isMaintenanceTaskConfigured)); + break; + } + } + } + } + + @Test + public void setIsMaintenanceTaskRunning(){ + // Default status + assertFalse(config.getIsGitMaintenanceRunning()); + + config.setIsGitMaintenanceRunning(true); + assertTrue(config.getIsGitMaintenanceRunning()); + // When status is set to true + // Maintenance needs to be running + // assertTrue(config.getIsGitMaintenanceRunning()); + } + + @Test + public void checkValidCronSyntax() throws ANTLRException { + + MaintenanceTaskConfiguration.checkSanity("* * * * *"); + MaintenanceTaskConfiguration.checkSanity("1 * * * * "); + MaintenanceTaskConfiguration.checkSanity("H H(8-15)/2 * * 1-5"); + MaintenanceTaskConfiguration.checkSanity("H H 1,15 1-11 *"); + } + + @Test(expected = ANTLRException.class) + public void checkInvalidCronSyntax() throws ANTLRException{ + MaintenanceTaskConfiguration.checkSanity(""); + MaintenanceTaskConfiguration.checkSanity("*****"); + MaintenanceTaskConfiguration.checkSanity("a * * 1 *"); + } + + @Test + public void testGetGitVersion(){ + List gitVersion = MaintenanceTaskConfiguration.getGitVersion(); + assertThat("Version list size error", gitVersion.size(), is(greaterThan(1))); + assertThat("Major version out of range", gitVersion.get(0), is(both(greaterThan(0)).and(lessThan(99)))); + assertThat("Minor version out of range", gitVersion.get(1), is(both(greaterThan(-1)).and(lessThan(99)))); + if (gitVersion.size() > 2) { + assertThat("Patch version out of range", gitVersion.get(2), is(both(greaterThan(-1)).and(lessThan(99)))); + } + } +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/ParameterizedCronSyntaxTest.java b/src/test/java/jenkins/plugins/git/maintenance/ParameterizedCronSyntaxTest.java new file mode 100644 index 0000000000..78d4455ff6 --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/ParameterizedCronSyntaxTest.java @@ -0,0 +1,66 @@ +package jenkins.plugins.git.maintenance; + +import antlr.ANTLRException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.jvnet.hudson.test.JenkinsRule; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +@RunWith(Parameterized.class) +public class ParameterizedCronSyntaxTest { + + @ClassRule + public static JenkinsRule j = new JenkinsRule(); + String cronSyntax; + boolean isValid; + + TaskScheduler taskScheduler; + + public ParameterizedCronSyntaxTest(String cronSyntax,boolean isValid){ + this.cronSyntax = cronSyntax; + this.isValid = isValid; + taskScheduler = new TaskScheduler(); + + } + + @Parameterized.Parameters(name = "{0}") + public static Collection permuteCronSyntax(){ + List crons = new ArrayList<>(); + // valid cron syntax + crons.add(new Object[]{"H * * * *",true}); + crons.add(new Object[]{"* * * * *", true}); + crons.add(new Object[]{"@hourly",true}); + crons.add(new Object[]{"@weekly",true}); + crons.add(new Object[]{"@daily",true}); + crons.add(new Object[]{"H H 1,15 1-11 *",true}); + + // invalid cron syntax; + crons.add(new Object[]{"",false}); + crons.add(new Object[]{"**", false}); + crons.add(new Object[]{"60 1 1 1 1",false}); + crons.add(new Object[]{"1 1 1 1 9",false}); + crons.add(new Object[]{"1 24 32 11 5",false}); + crons.add(new Object[]{"",false}); + return crons; + } + + @Test + public void testCorrectAndIncorrectSyntaxInput(){ + try { + assertNotNull(taskScheduler.getCronTabList(cronSyntax)); + assertTrue(isValid); + }catch(ANTLRException e){ + assertFalse(isValid); + } + } + +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/TaskExecutorTest.java b/src/test/java/jenkins/plugins/git/maintenance/TaskExecutorTest.java new file mode 100644 index 0000000000..249dddf541 --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/TaskExecutorTest.java @@ -0,0 +1,161 @@ +package jenkins.plugins.git.maintenance; + +import hudson.Launcher; +import hudson.model.TaskListener; +import hudson.plugins.git.*; +import hudson.util.StreamTaskListener; +import jenkins.model.GlobalConfiguration; +import jenkins.plugins.git.GitSampleRepoRule; +import jenkins.scm.api.SCMFileSystem; +import org.jenkinsci.plugins.gitclient.GitClient; +import org.junit.*; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.jvnet.hudson.test.JenkinsRule; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.locks.ReentrantLock; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +@RunWith(Parameterized.class) +public class TaskExecutorTest extends AbstractGitRepository { + + @ClassRule + public static JenkinsRule rule = new JenkinsRule(); + + private Task gitTask; + + + public TaskExecutorTest(TaskType taskType){ + this.gitTask = new Task(taskType); + this.gitTask.setCronSyntax("* * * * *"); + this.gitTask.setIsTaskConfigured(true); + } + + @Parameterized.Parameters(name = "{0}") + public static Collection permuteMaintenanceTasks(){ + + List maintenanceTasks = new ArrayList<>(); + + maintenanceTasks.add(new Object[]{TaskType.PREFETCH}); + maintenanceTasks.add(new Object[]{TaskType.GC}); + maintenanceTasks.add(new Object[]{TaskType.LOOSE_OBJECTS}); + maintenanceTasks.add(new Object[]{TaskType.INCREMENTAL_REPACK}); + maintenanceTasks.add(new Object[]{TaskType.COMMIT_GRAPH}); + return maintenanceTasks; + } + + @Test + public void testGitClient(){ + // Get directory of a single cache. + TaskExecutor taskExecutor = new TestTaskExecutorHelper(gitTask,testGitDir); + assertTrue(taskExecutor.getCaches().size() > 0); + GitMaintenanceSCM.Cache cache = taskExecutor.getCaches().get(0); + File cacheFile = cache.getCacheFile(); + assertNotNull(taskExecutor.getGitClient(cacheFile)); + assertThat(taskExecutor.getGitClient(cacheFile),instanceOf(GitClient.class)); + } + + @Test + public void testNullFileInGetGitClient() { + GitClient client = new TestTaskExecutorHelper(gitTask,null).getGitClient(null); + assertNull(client); + } + + @Test + public void testGetCaches(){ + TaskExecutor taskExecutor = new TestTaskExecutorHelper(gitTask,testGitDir); + assertNotNull(taskExecutor.getCaches()); + } + + @Test + public void testExecuteGitMaintenance() throws Exception { + + // This will create a pack file, incremental repack will then start to work. + repo.git("repack"); + + TaskExecutor taskExecutor = new TestTaskExecutorHelper(gitTask,testGitDir); + GitMaintenanceSCM.Cache cache = taskExecutor.getCaches().get(0); + File cacheFile = cache.getCacheFile(); + GitClient client = taskExecutor.getGitClient(cacheFile); + boolean isExecuted = taskExecutor.executeGitMaintenance(client,gitTask.getTaskType()); + + // based on the underlying git version it will work. + // If git version < 2.30, tests may fail. + assertThat(isExecuted,is(true)); + } + + @Test + public void testRunnable() throws Exception { + + // This will create a pack file, incremental repack will then start to work. + repo.git("repack"); + + MaintenanceTaskConfiguration config = GlobalConfiguration.all().get(MaintenanceTaskConfiguration.class); + config.setIsGitMaintenanceRunning(true); + config.setCronSyntax(gitTask.getTaskType(),"* * * * *"); + config.setIsTaskConfigured(gitTask.getTaskType(),true); + + TaskScheduler scheduler = new TaskScheduler(); + scheduler.scheduleTasks(); + } +// + // Todo Need a way to test termination of execution thread. + +// @Test +// public void testTerminateThread(){ +// +// } + + class TestTaskExecutorHelper extends TaskExecutor { + + File testGitDir; + public TestTaskExecutorHelper(Task maintenanceTask,File testGitDir) { + super(maintenanceTask); + this.testGitDir = testGitDir; + } + + List getCaches(){ + List caches = new ArrayList<>(); + caches.add(new GitMaintenanceSCM.Cache(testGitDir,new ReentrantLock())); + return caches; + } + } + + static List getGitVersion(){ + + final TaskListener procListener = StreamTaskListener.fromStderr(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + new Launcher.LocalLauncher(procListener).launch().cmds("git", "--version").stdout(out).join(); + } catch (IOException | InterruptedException ex) { + throw new RuntimeException("Couldn't fetch git maintenance command"); + } + String versionOutput = ""; + try { + versionOutput = out.toString(StandardCharsets.UTF_8.toString()).trim(); + } catch (UnsupportedEncodingException ue) { + throw new RuntimeException("Unsupported encoding version"); + } + final String[] fields = versionOutput.split(" ")[2].replaceAll("msysgit.", "").replaceAll("windows.", "").split("\\."); + + // Eg: [2, 31, 4] + // 0th index is Major Version. + // 1st index is Minor Version. + // 2nd index is Patch Version. + return Arrays.stream(fields).map(Integer::parseInt).collect(Collectors.toList()); + } +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/TaskSchedulerTest.java b/src/test/java/jenkins/plugins/git/maintenance/TaskSchedulerTest.java new file mode 100644 index 0000000000..1b9937fe3c --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/TaskSchedulerTest.java @@ -0,0 +1,165 @@ +package jenkins.plugins.git.maintenance; + +import jenkins.model.GlobalConfiguration; +import jenkins.plugins.git.GitSampleRepoRule; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class TaskSchedulerTest { + + @Rule + public JenkinsRule j = new JenkinsRule(); + + @Rule + public GitSampleRepoRule sampleRepo1 = new GitSampleRepoRule(); + + private TaskScheduler taskScheduler; + private MaintenanceTaskConfiguration config; + + @Before + public void setUp() throws Exception { + taskScheduler = new TaskScheduler(); + config = new MaintenanceTaskConfiguration(); + } + + // Tested all the internal functions of this method + @Test + public void testScheduleTasks() { + config.setIsGitMaintenanceRunning(true); + config.setCronSyntax(TaskType.PREFETCH,"* * * * *"); + config.setIsTaskConfigured(TaskType.PREFETCH,true); + taskScheduler.scheduleTasks(); + } + + @Test + public void testCheckIsTaskInQueue(){ + config.setCronSyntax(TaskType.PREFETCH,"* * * * *"); + config.setIsTaskConfigured(TaskType.PREFETCH,true); + + config.setCronSyntax(TaskType.COMMIT_GRAPH,"* * * * *"); + config.setIsTaskConfigured(TaskType.COMMIT_GRAPH,true); + + List maintenanceTasks = config.getMaintenanceTasks(); + taskScheduler.addTasksToQueue(maintenanceTasks); + + List configuredTask = config.getMaintenanceTasks().stream().filter(Task::getIsTaskConfigured).collect(Collectors.toList()); + + configuredTask.forEach(task -> assertTrue(taskScheduler.checkIsTaskInQueue(task))); + } + + @Test + public void testAddTasksToQueue() { + // Adding Maintenance tasks configuration; + config.setCronSyntax(TaskType.PREFETCH,"* * * * *"); + config.setIsTaskConfigured(TaskType.PREFETCH,true); + config.setCronSyntax(TaskType.LOOSE_OBJECTS,"* * * * *"); + config.setIsTaskConfigured(TaskType.LOOSE_OBJECTS,true); + + config.setCronSyntax(TaskType.GC,"5 1 1 1 1"); + config.setIsTaskConfigured(TaskType.GC,true); + config.setCronSyntax(TaskType.COMMIT_GRAPH,"H * * * *"); + List maintenanceTasks = config.getMaintenanceTasks(); + + int length = 2; + + taskScheduler.addTasksToQueue(maintenanceTasks); + assertThat(taskScheduler.getMaintenanceQueue().size(),is(length)); + } + + @Test + public void testInvalidAddTasksToQueue() { + config.setCronSyntax(TaskType.PREFETCH,"*****"); + config.setIsTaskConfigured(TaskType.PREFETCH,true); + List maintenanceTasks = config.getMaintenanceTasks(); + taskScheduler.addTasksToQueue(maintenanceTasks); + assertThat(taskScheduler.getMaintenanceQueue().size(),is(0)); + } + + @Test + public void testIsGitMaintenanceTaskRunning(){ + // Setting value to true + config.setIsGitMaintenanceRunning(true); + boolean isGitMaintenanceTaskRunning = taskScheduler.isGitMaintenanceTaskRunning(config); + assertTrue(isGitMaintenanceTaskRunning); + + // set value to false + config.setIsGitMaintenanceRunning(false); + isGitMaintenanceTaskRunning = taskScheduler.isGitMaintenanceTaskRunning(config); + assertFalse(isGitMaintenanceTaskRunning); + } + + @Test + public void testCreateNoExecutorThread(){ + config.setCronSyntax(TaskType.PREFETCH,"5 1 1 1 1"); + config.setIsTaskConfigured(TaskType.PREFETCH,true); + + List maintenanceTasks = config.getMaintenanceTasks(); + taskScheduler.addTasksToQueue(maintenanceTasks); + taskScheduler.createTaskExecutorThread(); + assertNull(taskScheduler.getTaskExecutor()); + + } + + @Test + public void testCreateExecutionThread(){ + + config.setCronSyntax(TaskType.PREFETCH,"* * * * *"); + config.setIsTaskConfigured(TaskType.PREFETCH,true); + config.setCronSyntax(TaskType.LOOSE_OBJECTS,"* * * * *"); + config.setIsTaskConfigured(TaskType.LOOSE_OBJECTS,true); + config.setCronSyntax(TaskType.COMMIT_GRAPH,"* * * * *"); + config.setIsTaskConfigured(TaskType.COMMIT_GRAPH,true); + config.setCronSyntax(TaskType.GC,"* * * * *"); + config.setIsTaskConfigured(TaskType.GC,true); + + List maintenanceTasks = config.getMaintenanceTasks(); + taskScheduler.addTasksToQueue(maintenanceTasks); + taskScheduler.createTaskExecutorThread(); + assertTrue(taskScheduler.getTaskExecutor().isAlive()); + } + + @Test + public void testTerminateMaintenanceTask(){ + taskScheduler.terminateMaintenanceTaskExecution(); + assertNull(taskScheduler.getTaskExecutor()); + assertEquals(0,taskScheduler.getMaintenanceQueue().size()); + } + + // Need to revist this test +// @Test +// public void testTerminateMaintenanceTaskDuringThreadExecution() throws Exception { +// config.setCronSyntax(TaskType.PREFETCH,"* * * * *"); +// config.setIsTaskConfigured(TaskType.PREFETCH,true); +// // Need to add few caches to test if the Thread is being terminated... +// +// sampleRepo1.init(); +// sampleRepo1.git("checkout", "-b", "bug/JENKINS-42817"); +// sampleRepo1.write("file", "modified"); +// sampleRepo1.git("commit", "--all", "--message=dev"); +// +// SCMFileSystem.of(j.createFreeStyleProject(), new GitSCM(GitSCM.createRepoList(sampleRepo1.toString(), null), Collections.singletonList(new BranchSpec("*/bug/JENKINS-42817")), null, null, Collections.emptyList())); +// +// List tasks = config.getMaintenanceTasks(); +// taskScheduler.addTasksToQueue(tasks); +// taskScheduler.createTaskExecutorThread(); +// +// assertTrue(taskScheduler.getTaskExecutor().isAlive()); +// taskScheduler.terminateMaintenanceTaskExecution(); +// // This test could depend on CPU speed. Faster execution can fail the test. +// Thread.sleep(1); +// assertFalse(taskScheduler.getTaskExecutor().isAlive()); +// } + +} diff --git a/src/test/java/jenkins/plugins/git/maintenance/TaskTest.java b/src/test/java/jenkins/plugins/git/maintenance/TaskTest.java new file mode 100644 index 0000000000..0a96aa9d8d --- /dev/null +++ b/src/test/java/jenkins/plugins/git/maintenance/TaskTest.java @@ -0,0 +1,97 @@ +package jenkins.plugins.git.maintenance; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +@RunWith(Parameterized.class) +public class TaskTest { + + private final Task task; + private final TaskType taskType; + private final String taskName; + + public TaskTest(Task task, TaskType taskType, String taskName) { + this.task = task; + this.taskType = taskType; + this.taskName = taskName; + } + + @Parameterized.Parameters(name = "{0}") + public static Collection permuteTasks() { + List tasks = new ArrayList<>(); + for (TaskType taskType : TaskType.values()) { + Object[] task = {new Task(taskType), taskType, taskType.getTaskName()}; + tasks.add(task); + } + + return tasks; + } + + @Test + public void getTaskType() { + assertThat(task.getTaskType(), is(taskType)); + } + + @Test + public void getTaskName() { + assertThat(task.getTaskName(), is(taskName)); + } + + @Test + public void setIsConfigured() { + task.setIsTaskConfigured(true); + assertTrue(task.getIsTaskConfigured()); + } + + @Test + public void checkCronSyntax(){ + String cronSyntax = "* * 1 * 1"; + task.setCronSyntax(cronSyntax); + assertThat(task.getCronSyntax(), is(cronSyntax)); + } + + @Test + public void testGetTaskType() { + assertThat(task.getTaskType(), is(taskType)); + } + + @Test + public void testGetTaskName() { + assertThat(task.getTaskName(), is(taskName)); + } + + @Test + public void testGetCronSyntax(){ + String cronSyntax = "* * 1 * 1"; + task.setCronSyntax(cronSyntax); + assertThat(task.getCronSyntax(), is(cronSyntax)); + } + + @Test + public void testSetIsTaskConfigured() { + task.setIsTaskConfigured(true); + assertTrue(task.getIsTaskConfigured()); + } + + @Test + public void testGetIsTaskConfigured() { + task.setIsTaskConfigured(true); + assertTrue(task.getIsTaskConfigured()); + } + + @Test + public void testGetIsTaskConfiguredFalse() { + task.setIsTaskConfigured(false); + assertFalse(task.getIsTaskConfigured()); + } +}