Drop file continously from cache as we read it.

container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java

@@ -2,7 +2,6 @@
 package com.yahoo.container.logging;
 
 import com.yahoo.concurrent.ThreadFactoryFactory;
-import com.yahoo.container.core.AccessLogConfig;
 import com.yahoo.io.NativeIO;
 import com.yahoo.log.LogFileDb;
 import com.yahoo.system.ProcessExecuter;
@@ -41,8 +40,6 @@ public class LogFileHandler extends StreamHandler {
     private final boolean compressOnRotation;
     private long[] rotationTimes = {0}; //default to one log per day, at midnight
     private String filePattern = "./log.%T";  // default to current directory, ms time stamp
-    private long lastRotationTime = -1; // absolute time (millis since epoch) of current file start
-    private int numberOfRecords = -1;
     private long nextRotationTime = 0;
     private OutputStream currentOutputStream = null;
     private String fileName;
@@ -122,7 +119,7 @@ public void publish(LogRecord r) {
         }
     }
 
-    private void internalPublish(LogRecord r) throws InterruptedException {
+    private void internalPublish(LogRecord r) {
         // first check to see if new file needed.
         // if so, use this.internalRotateNow() to do it
 
@@ -133,8 +130,6 @@ private void internalPublish(LogRecord r) throws InterruptedException {
         if (now > nextRotationTime || currentOutputStream == null) {
             internalRotateNow();
         }
-        // count records, and publish
-        numberOfRecords++;
         super.publish(r);
     }
 
@@ -177,9 +172,9 @@ long getNextRotationTime (long now) {
         }
         long nowTod = timeOfDayMillis(now);
         long next = 0;
-        for (int i = 0; i<rotationTimes.length; i++) {
-            if (nowTod < rotationTimes[i]) {
-                next = rotationTimes[i]-nowTod + now;
+        for (long rotationTime : rotationTimes) {
+            if (nowTod < rotationTime) {
+                next = rotationTime-nowTod + now;
                 break;
             }
         }
@@ -220,7 +215,7 @@ void rotateNow () {
 
     // Throw InterruptedException upwards rather than relying on isInterrupted to stop the thread as
     // isInterrupted() returns false after interruption in p.waitFor
-    private void internalRotateNow() throws InterruptedException {
+    private void internalRotateNow() {
         // figure out new file name, then
         // use super.setOutputStream to switch to a new file
 
@@ -243,8 +238,6 @@ private void internalRotateNow() throws InterruptedException {
 
         createSymlinkToCurrentFile();
 
-        numberOfRecords = 0;
-        lastRotationTime = now;
         nextRotationTime = 0; //figure it out later (lazy evaluation)
         if ((oldFileName != null)) {
             File oldFile = new File(oldFileName);
@@ -259,21 +252,24 @@ private void internalRotateNow() throws InterruptedException {
         }
     }
 
+
     private void runCompression(File oldFile) {
         File gzippedFile = new File(oldFile.getPath() + ".gz");
         try (GZIPOutputStream compressor = new GZIPOutputStream(new FileOutputStream(gzippedFile), 0x100000);
              FileInputStream inputStream = new FileInputStream(oldFile))
         {
-            byte [] buffer = new byte[0x100000];
+            byte [] buffer = new byte[0x400000]; // 4M buffer
 
+            NativeIO nativeIO = new NativeIO();
+            long totalBytesRead = 0;
             for (int read = inputStream.read(buffer); read > 0; read = inputStream.read(buffer)) {
                 compressor.write(buffer, 0, read);
+                nativeIO.dropPartialFileFromCache(inputStream.getFD(), totalBytesRead, read, false);
+                totalBytesRead += read;
             }
             compressor.finish();
             compressor.flush();
 
-            NativeIO nativeIO = new NativeIO();
-            nativeIO.dropFileFromCache(oldFile); // Drop from cache in case somebody else has a reference to it preventing from dying quickly.
             oldFile.delete();
             nativeIO.dropFileFromCache(gzippedFile);
         } catch (IOException e) {
@@ -306,28 +302,6 @@ private void createSymlinkToCurrentFile() {
         }
     }
 
-    /**
-     * Name the current file to "name.n" where n
-     * 1+ the largest integer in existing file names
-     */
-    private void moveCurrentFile() {
-        File file=new File(fileName);
-        if ( ! file.exists()) return; // no current file
-        File dir=file.getParentFile();
-        Pattern logFilePattern=Pattern.compile(".*\\.(\\d+)");
-        long largestN=0;
-        for (File existingFile : dir.listFiles()) {
-            Matcher matcher=logFilePattern.matcher(existingFile.getName());
-            if (!matcher.matches()) continue;
-            long thisN=Long.parseLong(matcher.group(1));
-            if (thisN>largestN)
-                largestN=thisN;
-        }
-        File newFn = new File(dir, file.getName() + "." + (largestN + 1));
-        LogFileDb.nowLoggingTo(newFn.getAbsolutePath());
-        file.renameTo(newFn);
-    }
-
     /**
      * Calculate rotation times array, given times in minutes, as "0 60 ..."
      *

vespajlib/src/main/java/com/yahoo/io/NativeIO.java

@@ -54,19 +54,28 @@ public Throwable getError() {
     }
 
     /**
-     * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache.
+     * Will hint the OS that data read so far will not be accessed again and should hence be dropped from the buffer cache.
      * @param fd The file descriptor to drop from buffer cache.
      */
-    public void dropFileFromCache(FileDescriptor fd) {
-        try {
-            fd.sync();
-        } catch (SyncFailedException e) {
-            logger.warning("Sync failed while dropping cache: " + e.getMessage());
+    public void dropPartialFileFromCache(FileDescriptor fd, long offset, long len, boolean sync) {
+        if (sync) {
+            try {
+                fd.sync();
+            } catch (SyncFailedException e) {
+                logger.warning("Sync failed while dropping cache: " + e.getMessage());
+            }
         }
         if (initialized) {
-            posix_fadvise(getNativeFD(fd), 0, 0, POSIX_FADV_DONTNEED);
+            posix_fadvise(getNativeFD(fd), offset, len, POSIX_FADV_DONTNEED);
         }
     }
+    /**
+     * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache.
+     * @param fd The file descriptor to drop from buffer cache.
+     */
+    public void dropFileFromCache(FileDescriptor fd) {
+        dropPartialFileFromCache(fd, 0, 0, true);
+    }
 
     /**
      * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache.