Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Output log chunking #1722

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Output log chunking #1722

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 42 additions & 6 deletions src/searchd.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,11 +436,47 @@ void sphLog ( ESphLogLevel eLevel, const char * sFmt, va_list ap )
// format the message
if ( sFmt )
{
// need more space for tail zero and "\n" that added at sphLogEntry
int iSafeGap = 4;
int iBufSize = sizeof(sBuf)-iLen-iSafeGap;
vsnprintf ( sBuf+iLen, iBufSize, sFmt, ap );
sBuf[ sizeof(sBuf)-iSafeGap ] = '\0';
// Use vsnprintf to get the total length required for the formatted string
va_list ap_copy;
va_copy(ap_copy, ap);
int iRequiredLength = vsnprintf(NULL, 0, sFmt, ap_copy);
va_end(ap_copy);

// Allocate a temporary buffer to hold the entire formatted message
char* sMessage = (char*)malloc(iRequiredLength + 1); // +1 for null terminator
if (!sMessage)
{
// Handle malloc failure if necessary
return;
}

// Use vsnprintf again to write the formatted message into sMessage
va_copy(ap_copy, ap);
vsnprintf(sMessage, iRequiredLength + 1, sFmt, ap_copy);
va_end(ap_copy);

// Now iterate over the message, writing it in chunks
for (int iOffset = 0; iOffset < iRequiredLength;)
{
// Calculate space left in sBuf after the banner and timestamp
int iSpaceLeft = sizeof(sBuf) - iLen - 1; // -1 for null terminator

// Determine the chunk length (do not exceed the space left in sBuf)
int iChunkLength = (iRequiredLength - iOffset) < iSpaceLeft ? (iRequiredLength - iOffset) : iSpaceLeft;

// Copy the next chunk of the message into the buffer
strncpy(sBuf + iLen, sMessage + iOffset, iChunkLength);
sBuf[iLen + iChunkLength] = '\0'; // Ensure null-termination

// Log the current chunk
sphLogEntry(eLevel, sBuf, sTtyBuf);

// Move the offset by the chunk length for the next iteration
iOffset += iChunkLength;
}

// Free the allocated memory for the message
free(sMessage);
}

if ( sFmt && eLevel>SPH_LOG_INFO && g_iLogFilterLen )
Expand Down Expand Up @@ -490,7 +526,7 @@ void sphLog ( ESphLogLevel eLevel, const char * sFmt, va_list ap )
uLastEntry = uEntry;

// do the logging
sphLogEntry ( eLevel, sBuf, sTtyBuf );
// sphLogEntry ( eLevel, sBuf, sTtyBuf );
}

void Shutdown (); // forward
Expand Down
88 changes: 38 additions & 50 deletions src/searchdbuddy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,56 +133,44 @@ static void AddTail ( Str_t tLine )
memcpy ( pDst, tLine.first, tLine.second );
}

static bool HasLineEnd ( Str_t tBuf, Str_t tLine )
{
const char * sEnd = tBuf.first + tBuf.second;
return ( ( tLine.first + tLine.second )<sEnd );
}

static void LogPipe ( Str_t tSrc )
{
CSphVector<Str_t> dLines;
sphSplitApply ( tSrc.first, tSrc.second, "\n\r", [&dLines] ( const char * pLine, int iLen ) { dLines.Add ( Str_t { pLine, iLen } ); } );

if ( !dLines.GetLength() )
return;
static bool HasLineEnd(Str_t tBuf, Str_t tLine) {
if (tLine.second == 0) {
// Empty line, no content to check for line ending
return false;
}

// whole pipe buffer without line end - collect into line buffer
Str_t tLine0 = dLines[0];
if ( !HasLineEnd ( tSrc, tLine0 ) )
{
AddTail ( tLine0 );
return;
}
// Get the last character of the line
const char lastChar = *(tLine.first + tLine.second - 1);
// Check if the last character is a line-ending character
return lastChar == '\n' || lastChar == '\r';
}

// join pipe buffer with line buffer collected so far
if ( g_dLogBuf.GetLength() )
{
sphInfo ( "[BUDDY] %.*s%.*s", g_dLogBuf.GetLength(), g_dLogBuf.Begin(), tLine0.second, tLine0.first );
g_dLogBuf.Resize ( 0 );
} else
{
sphInfo ( "[BUDDY] %.*s", tLine0.second, tLine0.first );
}
static void LogPipe(Str_t tSrc) {
CSphVector<Str_t> dLines;
sphSplitApply(tSrc.first, tSrc.second, "\n\r", [&dLines](const char* pLine, int iLen) {
dLines.Add(Str_t{pLine, iLen});
});

if ( dLines.GetLength()==1 )
return;
if (!dLines.GetLength()) {
return;
}

for ( int i=1; i<dLines.GetLength()-1; i++ )
{
Str_t tLine = dLines[i];
sphInfo ( "[BUDDY] %.*s", tLine.second, tLine.first );
}
// Process each line and log it immediately
for (int i = 0; i < dLines.GetLength(); ++i) {
Str_t& tLine = dLines[i];
bool hasLineEnd = HasLineEnd(tSrc, tLine);

// If this line does not have a line end, it may be part of the next chunk
if (!hasLineEnd && i == dLines.GetLength() - 1) {
AddTail(tLine); // Save incomplete line for next chunk processing
} else {
// Log the line immediately
sphInfo("[BUDDY] %.*s", tLine.second, tLine.first);
}
}

Str_t tLineLast = dLines.Last();
// last line could be without line end - collect into line buffer
if ( HasLineEnd ( tSrc, tLineLast ) )
{
sphInfo ( "[BUDDY] %.*s", tLineLast.second, tLineLast.first );
} else
{
AddTail ( tLineLast );
}
// Clear the log buffer, since we are logging each line immediately
g_dLogBuf.Resize(0);
}

static void ReadFromPipe ( const boost::system::error_code & tGotCode, std::size_t iSize )
Expand Down Expand Up @@ -327,7 +315,7 @@ BuddyState_e TryToStart ( const char * sArgs, CSphString & sError )
g_pIOS->stop();

g_pPipe.reset();
g_pIOS.reset();
g_pIOS.reset();

g_pIOS.reset ( new boost::asio::io_service );
g_pPipe.reset ( new boost::process::async_pipe ( *g_pIOS ) );
Expand Down Expand Up @@ -361,7 +349,7 @@ CSphString GetUrl ( const ListenerDesc_t & tDesc )
{
char sAddrBuf [ SPH_ADDRESS_SIZE ];
sphFormatIP ( sAddrBuf, sizeof(sAddrBuf), tDesc.m_uIP );

CSphString sURI;
sURI.SetSprintf ( "http://%s:%d", sAddrBuf, tDesc.m_iPort );

Expand Down Expand Up @@ -414,7 +402,7 @@ void BuddyStart ( const CSphString & sConfigPath, bool bHasBuddyPath, const VecT
g_sListener4Buddy.cstr(),
( bTelemetry ? "" : "--disable-telemetry" ),
iThreads );


CSphString sErorr;
BuddyState_e eBuddy = TryToStart ( g_sStartArgs.cstr(), sErorr );
Expand Down Expand Up @@ -559,7 +547,7 @@ bool ProcessHttpQueryBuddy ( HttpProcessResult_t & tRes, Str_t sSrcQuery, Option
tRes.m_sError.SetSprintf ( "can not process /cli endpoint with User-Agent:Manticore Buddy" );
sphHttpErrorReply ( dResult, SPH_HTTP_STATUS_501, tRes.m_sError.cstr() );
}

assert ( dResult.GetLength()>0 );
return tRes.m_bOk;
}
Expand Down Expand Up @@ -683,4 +671,4 @@ CSphString BuddyGetPath ( const CSphString & sConfigPath, bool bHasBuddyPath )
sphWarning ( "[BUDDY] no %s found at '%s', disabled", g_sDefaultBuddyName.cstr(), sPathBuddy2Module.cstr() );

return CSphString();
}
}
2 changes: 0 additions & 2 deletions src/sphinxutils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,6 @@ static KeyDesc_t g_dKeysIndex[] =
{ "access_blob_attrs", 0, nullptr },
{ "access_doclists", 0, nullptr },
{ "access_hitlists", 0, nullptr },
{ "access_dict", 0, nullptr },
{ "stored_fields", 0, nullptr },
{ "stored_only_fields", 0, nullptr },
{ "docstore_block_size", 0, nullptr },
Expand Down Expand Up @@ -1035,7 +1034,6 @@ static KeyDesc_t g_dKeysSearchd[] =
{ "access_blob_attrs", 0, nullptr },
{ "access_doclists", 0, nullptr },
{ "access_hitlists", 0, nullptr },
{ "access_dict", 0, nullptr },
{ "docstore_cache_size", 0, nullptr },
{ "skiplist_cache_size", 0, nullptr },
{ "ssl_cert", 0, nullptr },
Expand Down