logfile: exhance internal dlt logging by introducing size limits (#369)

Enhance dlt logging such that multiple files are used as it is done
for the offline traces. Add limit-specific config values for logging.
For this purpose the pattern of index-based file names is used only.
This approach of logging to multiple files and rotating in order
to keep the limits ensures that dlt logs take care of available
space on the underlying file system and do not grow infinitely.

Signed-off-by: Daniel Weber <[email protected]>
Co-authored-by: Oleg Tropmann <[email protected]>

doc/dlt.conf.5.md

@@ -79,6 +79,26 @@ If LoggingMode is set to 2 logs are written to the file path given here.
 
     Default: /tmp/dlt.log
 
+## EnableLoggingFileLimit
+
+Only relevant for logging in file (LoggingMode = 2).
+If EnableLoggingFileLimit is set to 0, the daemon logs to one logging file without any size limit.
+If EnableLoggingFileLimit is set to 1, the daemon considers the size limits configured by LoggingFileSize and LoggingFileMaxSize. If the limits are configured accordingly, multiple log files are used.
+
+    Default: 0
+
+## LoggingFileSize
+
+Only considered for logging in file (LoggingMode = 2) and EnableLoggingFileLimit = 1. Maximum size in bytes of one logging file.
+
+    Default: 250000
+
+## LoggingFileMaxSize
+
+Only considered for logging in file (LoggingMode = 2) and EnableLoggingFileLimit = 1. Maximum size in bytes of all logging files.
+
+    Default: 1000000
+
 ## TimeOutOnSend
 
 Socket timeout in seconds for sending to clients.

include/dlt/dlt_common.h

@@ -191,13 +191,13 @@
 #      define LOG_DAEMON  (3 << 3)
 #   endif
 
-enum {
+typedef enum {
     DLT_LOG_TO_CONSOLE = 0,
     DLT_LOG_TO_SYSLOG = 1,
     DLT_LOG_TO_FILE = 2,
     DLT_LOG_TO_STDERR = 3,
     DLT_LOG_DROPPED = 4
-};
+} DltLoggingMode;
 
 /**
  * The standard TCP Port used for DLT daemon, can be overwritten via -p \<port\> when starting dlt-daemon
@@ -1670,6 +1670,57 @@ void dlt_hex_ascii_to_binary(const char *ptr, uint8_t *binary, int *size);
  */
 int dlt_execute_command(char *filename, char *command, ...);
 
+/**
+ * Return the extension of given file name.
+ * @param filename Only file names without prepended path allowed.
+ * @return pointer to extension
+ */
+char *get_filename_ext(const char *filename);
+
+/**
+ * Extract the base name of given file name (without the extension).
+ * @param abs_file_name Absolute path to file name.
+ * @param base_name Base name it is extracted to.
+ * @param base_name_length Base name length.
+ * @return indicating success
+ */
+bool dlt_extract_base_name_without_ext(const char* const abs_file_name, char* base_name, long base_name_len);
+
+/**
+ * Initialize (external) logging facility
+ * @param mode DltLoggingMode, 0 = log to stdout, 1 = log to syslog, 2 = log to file, 3 = log to stderr
+ * @param enable_multiple_logfiles, true if multiple logfiles (incl. size limits) should be use
+ * @param logging_file_size, maximum size in bytes of one logging file
+ * @param logging_files_max_size, maximum size in bytes of all logging files
+ */
+DltReturnValue dlt_log_init_multiple_logfiles_support(DltLoggingMode mode, bool enable_multiple_logfiles, int logging_file_size, int logging_files_max_size);
+
+/**
+ * Initialize (external) logging facility for single logfile.
+ */
+DltReturnValue dlt_log_init_single_logfile();
+
+/**
+ * Initialize (external) logging facility for multiple files logging.
+ */
+DltReturnValue dlt_log_init_multiple_logfiles(int logging_file_size, int logging_files_max_size);
+
+/**
+ * Logs into log files represented by the multiple files buffer.
+ * @param format First element in a specific format that will be logged.
+ * @param ... Further elements in a specific format that will be logged.
+ */
+void dlt_log_multiple_files_write(const char* format, ...);
+
+void dlt_log_free_single_logfile();
+
+void dlt_log_free_multiple_logfiles();
+
+/**
+ * Checks whether (internal) logging in multiple files is active.
+ */
+bool dlt_is_log_in_multiple_files_active();
+
 #   ifdef __cplusplus
 }
 #   endif

include/dlt/dlt_multiple_files.h

@@ -0,0 +1,154 @@
+/*
+ * SPDX license identifier: MPL-2.0
+ *
+ * Copyright (C) 2011-2015, BMW AG
+ *
+ * This file is part of GENIVI Project DLT - Diagnostic Log and Trace.
+ *
+ * This Source Code Form is subject to the terms of the
+ * Mozilla Public License (MPL), v. 2.0.
+ * If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * For further information see http://www.genivi.org/.
+ */
+
+/*!
+ * \author
+ * Oleg Tropmann <[email protected]>
+ * Daniel Weber <[email protected]>
+ *
+ * \copyright Copyright © 2022 Mercedes-Benz AG. \n
+ * License MPL-2.0: Mozilla Public License version 2.0 http://mozilla.org/MPL/2.0/.
+ *
+ * \file dlt_multiple_files.h
+ */
+
+
+#ifndef DLT_MULTIPLE_FILES_H
+#define DLT_MULTIPLE_FILES_H
+
+#include <limits.h>
+
+#include "dlt_common.h"
+#include "dlt_types.h"
+
+#define MULTIPLE_FILES_FILENAME_INDEX_DELIM "."
+#define MULTIPLE_FILES_FILENAME_TIMESTAMP_DELIM "_"
+
+/**
+ * Represents a ring buffer of multiple files of identical file size.
+ * File names differ in timestamp or index (depending on chosen mode).
+ * This buffer is used, e.g. for dlt offline traces and the internal dlt logging (dlt.log)
+ */
+typedef struct
+{
+    char directory[NAME_MAX + 1];/**< (String) Store DLT messages to local directory */
+    char filename[NAME_MAX + 1]; /**< (String) Filename of currently used log file */
+    int fileSize;                /**< (int) Maximum size in bytes of one file, e.g. for offline trace 1000000 as default */
+    int maxSize;                 /**< (int) Maximum size of all files, e.g. for offline trace 4000000 as default */
+    bool filenameTimestampBased;  /**< (bool) is filename timestamp based? false = index based (Default: true) */
+    char filenameBase[NAME_MAX + 1];/**< (String) Prefix of file name */
+    char filenameExt[NAME_MAX + 1];/**< (String) Extension of file name */
+    int ohandle;                 /**< (int) file handle to current output file */
+} MultipleFilesRingBuffer;
+
+/**
+ * Initialise the multiple files buffer.
+ * This function call opens the currently used log file.
+ * A check of the complete size of the files is done during startup.
+ * Old files are deleted, if there is not enough space left to create new file.
+ * This function must be called before using further multiple files functions.
+ * @param files_buffer pointer to MultipleFilesRingBuffer struct.
+ * @param directory directory where to store multiple files.
+ * @param file_size maximum size of one files.
+ * @param max_size maximum size of complete multiple files in bytes.
+ * @param filename_timestamp_based filename to be created on timestamp-based or index-based.
+ * @param append Indicates whether the current log files is used or a new file should be be created
+ * @param filename_base Base name.
+ * @param filename_ext File extension.
+ * @return negative value if there was an error.
+ */
+extern DltReturnValue multiple_files_buffer_init(MultipleFilesRingBuffer *files_buffer,
+                                                 const char *directory,
+                                                 int file_size,
+                                                 int max_size,
+                                                 bool filename_timestamp_based,
+                                                 bool append,
+                                                 const char *filename_base,
+                                                 const char *filename_ext);
+
+/**
+ * Uninitialise the multiple files buffer.
+ * This function call closes currently used log file.
+ * This function must be called after usage of multiple files.
+ * @param files_buffer pointer to MultipleFilesRingBuffer struct.
+ * @return negative value if there was an error.
+*/
+extern DltReturnValue multiple_files_buffer_free(const MultipleFilesRingBuffer *files_buffer);
+
+/**
+ * Write data into multiple files.
+ * If the current used log file exceeds the max file size, new log file is created.
+ * A check of the complete size of the multiple files is done before new file is created.
+ * Old files are deleted, if there is not enough space left to create new file.
+ * @param files_buffer pointer to MultipleFilesRingBuffer struct.
+ * @param data pointer to first data block to be written, null if not used.
+ * @param size size in bytes of first data block to be written, 0 if not used.
+ * @return negative value if there was an error.
+ */
+extern DltReturnValue multiple_files_buffer_write(MultipleFilesRingBuffer *files_buffer,
+                                                  const unsigned char *data,
+                                                  int size);
+
+/**
+ * First the limits are verified. Then the oldest file is deleted and a new file is created on demand.
+ * @param files_buffer pointer to MultipleFilesRingBuffer struct.
+ * @param size size in bytes of data that will be written.
+ */
+void multiple_files_buffer_rotate_file(MultipleFilesRingBuffer *files_buffer,
+                                       int size);
+
+/**
+ * Writes the given data to current file specified by corresponding file handle.
+ * @param files_buffer pointer to MultipleFilesRingBuffer struct.
+ * @param data pointer to data block to be written, null if not used.
+ * @param size size in bytes of given data block to be written, 0 if not used.
+ */
+DltReturnValue multiple_files_buffer_write_chunk(const MultipleFilesRingBuffer *files_buffer,
+                                                 const unsigned char *data,
+                                                 int size);
+
+/**
+ * Get size of currently used multiple files buffer.
+ * @return size in bytes.
+ */
+extern ssize_t multiple_files_buffer_get_total_size(const MultipleFilesRingBuffer *files_buffer);
+
+/**
+ * Provides info about the multiple files storage directory.
+ * @param path path of the storage directory
+ * @param file_name filename to search for
+ * @param newest pointer to store newest filename
+ * @param oldest pointer to store oldest filename
+ * @return num of files in the directory.
+ */
+unsigned int multiple_files_buffer_storage_dir_info(const char *path, const char *file_name,
+                                                    char *newest, char *oldest);
+
+/**
+ * Creates filename with index.
+ * @param files_buffer pointer to MultipleFilesRingBuffer struct.
+ * @param length the maximum length of the log_file_name.
+ * @param idx index to be used for file name creation.
+ */
+void multiple_files_buffer_file_name(MultipleFilesRingBuffer *files_buffer, size_t length, unsigned int idx);
+
+/**
+ * Generates index for log file name.
+ * @param file filename supplied to create index.
+ * @return the index to be used for log file name.
+ */
+unsigned int multiple_files_buffer_get_idx_of_log_file(char *file);
+
+#endif // DLT_MULTIPLE_FILES_H

include/dlt/dlt_offline_trace.h

@@ -57,105 +57,32 @@
 
 #include <limits.h>
 
+#include "dlt_multiple_files.h"
 #include "dlt_types.h"
 
 #define DLT_OFFLINETRACE_FILENAME_BASE "dlt_offlinetrace"
-#define DLT_OFFLINETRACE_FILENAME_INDEX_DELI "."
-#define DLT_OFFLINETRACE_FILENAME_TIMESTAMP_DELI "_"
 #define DLT_OFFLINETRACE_FILENAME_EXT  ".dlt"
 
-typedef struct
-{
-    char directory[NAME_MAX + 1];/**< (String) Store DLT messages to local directory */
-    char filename[NAME_MAX + 1]; /**< (String) Filename of currently used log file */
-    int fileSize;                /**< (int) Maximum size in bytes of one trace file (Default: 1000000) */
-    int maxSize;                 /**< (int) Maximum size of all trace files (Default: 4000000) */
-    int filenameTimestampBased;  /**< (int) timestamp based or index based (Default: 1 Timestamp based) */
-    int ohandle;
-} DltOfflineTrace;
-
-/**
- * Initialise the offline trace
- * This function call opens the currently used log file.
- * A check of the complete size of the offline trace is done during startup.
- * Old files are deleted, if there is not enough space left to create new file.
- * This function must be called before using further offline trace functions.
- * @param trace pointer to offline trace structure
- * @param directory directory where to store offline trace files
- * @param fileSize maximum size of one offline trace file.
- * @param maxSize maximum size of complete offline trace in bytes.
- *.@param filenameTimestampBased filename to be created on timestamp based or index based
- * @return negative value if there was an error
- */
-extern DltReturnValue dlt_offline_trace_init(DltOfflineTrace *trace,
-                                             const char *directory,
-                                             int fileSize,
-                                             int maxSize,
-                                             int filenameTimestampBased);
-
 /**
- * Uninitialise the offline trace
- * This function call closes currently used log file.
- * This function must be called after usage of offline trace
- * @param buf pointer to offline trace structure
- * @return negative value if there was an error
- */
-extern DltReturnValue dlt_offline_trace_free(DltOfflineTrace *buf);
-
-/**
- * Write data into offline trace
+ * Write data into offline traces.
  * If the current used log file exceeds the max file size, new log file is created.
- * A check of the complete size of the offline trace is done before new file is created.
+ * A check of the complete size of the offline traces is done before new file is created.
  * Old files are deleted, if there is not enough space left to create new file.
- * @param trace pointer to offline trace structure
- * @param data1 pointer to first data block to be written, null if not used
- * @param size1 size in bytes of first data block to be written, 0 if not used
- * @param data2 pointer to second data block to be written, null if not used
- * @param size2 size in bytes of second data block to be written, 0 if not used
- * @param data3 pointer to third data block to be written, null if not used
- * @param size3 size in bytes of third data block to be written, 0 if not used
- * @return negative value if there was an error
+ * @param trace pointer to MultipleFilesRingBuffer struct.
+ * @param data1 pointer to first data block to be written, null if not used.
+ * @param size1 size in bytes of first data block to be written, 0 if not used.
+ * @param data2 pointer to second data block to be written, null if not used.
+ * @param size2 size in bytes of second data block to be written, 0 if not used.
+ * @param data3 pointer to third data block to be written, null if not used.
+ * @param size3 size in bytes of third data block to be written, 0 if not used.
+ * @return negative value if there was an error.
  */
-extern DltReturnValue dlt_offline_trace_write(DltOfflineTrace *trace,
-                                              unsigned char *data1,
+extern DltReturnValue dlt_offline_trace_write(MultipleFilesRingBuffer *trace,
+                                              const unsigned char *data1,
                                               int size1,
-                                              unsigned char *data2,
+                                              const unsigned char *data2,
                                               int size2,
-                                              unsigned char *data3,
+                                              const unsigned char *data3,
                                               int size3);
 
-/**
- * Get size of currently used offline trace buffer
- * @return size in bytes
- */
-extern ssize_t dlt_offline_trace_get_total_size(DltOfflineTrace *trace);
-
-/**
- * Provides info about the offline logs storage directory
- * @param path path of the storage directory
- * @param file_name filename to search for
- * @param newest pointer to store newest filename
- * @param oldest pointer to store oldest filename
- * @return num of files in the directory
- */
-unsigned int dlt_offline_trace_storage_dir_info(char *path, char *file_name, char *newest, char *oldest);
-
-/**
- * creates filename with index
- * @param log_file_name file name created with index
- * @param length the maximum length of the log_file_name
- * @param name filename base
- * @param idx index to be used for file name creation
- */
-void dlt_offline_trace_file_name(char *log_file_name, size_t length,
-                                 char *name, unsigned int idx);
-
-/**
- * generates index for log file name
- * @param file filename supplied to create index
- * @return the index to be used for log file name
- */
-unsigned int dlt_offline_trace_get_idx_of_log_file(char *file);
-
-
 #endif /* DLT_OFFLINE_TRACE_H */

src/console/dlt-convert.c

@@ -126,17 +126,6 @@ void usage()
     printf("  -t            Handling input compressed files (tar.gz)\n");
 }
 
-char *get_filename_ext(const char *filename)
-{
-    if (filename == NULL)
-            fprintf(stderr, "ERROR: %s: invalid arguments\n", __FUNCTION__);
-
-    char *dot = strrchr(filename, '.');
-    if(!dot || dot == filename)
-        return "";
-    return dot + 1;
-}
-
 void empty_dir(const char *dir)
 {
     struct dirent **files = { 0 };