Skip to content

Commit

Permalink
fix: resume from reader cursor when no senders configured
Browse files Browse the repository at this point in the history
In cases where a reader was configured with no senders (eg. only
configured with searches), then in case of reloads/restarts the resume
cursor was assumed `None`, and the default 'initial_position' of 'head'
is fallen back to. This would end up causing all matching searchg to
resend metric events immediately.
  • Loading branch information
ngilles-aiven committed Mar 15, 2024
1 parent 3f80062 commit 14d8214
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 1 deletion.
6 changes: 5 additions & 1 deletion journalpump/journalpump.py
Original file line number Diff line number Diff line change
Expand Up @@ -838,7 +838,11 @@ def configure_readers(self):

for reader_name, reader_config in new_config.items():
reader_state = state.get("readers", {}).get(reader_name, {})
resume_cursor = None

# Default to reader's cursor, in case we have no senders
resume_cursor = reader_state.get("cursor")

# Check sender cursors
for sender_name, sender in reader_state.get("senders", {}).items():
sender_cursor = sender["sent"]["cursor"]
if sender_cursor is None:
Expand Down
46 changes: 46 additions & 0 deletions test/test_journalpump.py
Original file line number Diff line number Diff line change
Expand Up @@ -982,6 +982,52 @@ def test_log_level_filtering(self):
assert len(sender_d_msgs) == expected_results


def test_journalpump_resume_cursor(tmpdir) -> None:
journalpump_path = str(tmpdir.join("journalpump.json"))
statefile_path = str(tmpdir.join("journalpump_state.json"))

config = {
"json_state_file_path": statefile_path,
"readers": {
"without_senders": {"senders": {}},
"with_sender": {
"senders": {
"fake_syslog": {
"output_type": "rsyslog",
"rsyslog_server": "127.0.0.1",
"rsyslog_port": 514,
},
}
},
},
}

state = {
"readers": {
"without_senders": {
"cursor": "reader_cursor",
},
"with_sender": {
"cursor": "reader_cursor",
"senders": {"fake_syslog": {"sent": {"cursor": "sender_cursor"}}},
},
},
"start_time": datetime.now().isoformat()
}

with open(journalpump_path, "w") as fp:
fp.write(json.dumps(config))

with open(statefile_path, "w") as fp:
fp.write(json.dumps(state))

pump = JournalPump(journalpump_path)

# Check that readers are initialized with the right cursors
assert pump.readers["without_senders"].cursor == "reader_cursor"
assert pump.readers["with_sender"].cursor == "sender_cursor"


def test_journalpump_state_file(tmpdir):
journalpump_path = str(tmpdir.join("journalpump.json"))
statefile_path = str(tmpdir.join("journalpump_state.json"))
Expand Down

0 comments on commit 14d8214

Please sign in to comment.