Add pre-processing for realistic scenarios

[achamayou]

Extracting another bit of #5769, to move towards validating scenarios that fully match implementation behaviour.

This removes the replicate trace event that precedes an add_configuration to match ccfraft.tla behaviours.

[ghost]

trace_validation_prepare@78587 aka 20231109.10 vs main ewma over 20 builds from 78218 to 78568

Click to see table

main

build_id	build_number	pi_basic_mt_sgx_cft^	pi_basic_mt_sgx_cft_mem	Commit latency factor	tpcc_sgx_cft^	tpcc_sgx_cft_mem	ls_sgx_cft^	ls_sgx_cft_mem	pi_ls_sgx_cft^	pi_ls_sgx_cft_mem	pi_basic_sgx_cft^	pi_basic_sgx_cft_mem	pi_basic_mt_virtual_cft^	pi_basic_js_sgx_cft^	pi_basic_js_sgx_cft_mem	tpcc_virtual_cft^	ls_jwt_sgx_cft^	ls_jwt_sgx_cft_mem	ls_virtual_cft^	pi_ls_jwt_sgx_cft^	pi_ls_jwt_sgx_cft_mem	pi_ls_virtual_cft^	pi_basic_virtual_cft^	ls_js_sgx_cft^	ls_js_sgx_cft_mem	pi_basic_js_virtual_cft^	ls_full_js_sgx_cft^	ls_full_js_sgx_cft_mem	ls_jwt_virtual_cft^	pi_ls_jwt_virtual_cft^	ls_js_virtual_cft^	ls_js_jwt_sgx_cft^	ls_js_jwt_sgx_cft_mem	ls_full_js_virtual_cft^	ls_js_jwt_virtual_cft^	hist_sgx_cft^	RB put (/s)^	CHAMP put (/s)^	RB get (/s)^	CHAMP get (/s)^	tlc_3node_fixed_duration_s	tlc_3node_fixed_states	tlc_atomic_reconfig_duration_s	tlc_atomic_reconfig_states	tlc_reconfig_duration_s	tlc_reconfig_states
78218	20231101.11	27482.4	2.30851e+07	0.794466	5594.46	8.59996e+07	14033	1.88908e+07	14051.8	1.05021e+07	15531.3	1.46964e+07	83063.2	1428.2	1.25993e+07	17267	6867.98	1.67936e+07	43523.4	6933.2	6.30784e+06	39919.2	54609.1	5807.77	1.67936e+07	4338.1	5469.02	1.67936e+07	17157.6	18949.1	17376.4	3994.19	1.67936e+07	14871.1	10180.4	45431.3	833473	1.18618e+06	8.15072e+06	3.08689e+07	nan	nan	nan	nan	nan	nan
78226	20231101.13	27521.2	2.51822e+07	0.798772	5625.43	8.59996e+07	14043.1	1.88908e+07	14097.8	1.05021e+07	15604.3	1.46964e+07	73492.7	1414.9	1.25993e+07	17229.8	7241.73	1.67936e+07	43650.8	6889	6.30784e+06	46528.9	47448.2	5756.54	1.67936e+07	4341.1	5445.43	1.67936e+07	17155.8	18953.8	14940.7	3965.05	1.67936e+07	14721.7	10072.6	46340.6	835263	1.18362e+06	8.1689e+06	3.06211e+07	nan	nan	nan	nan	nan	nan
78246	20231101.20	27565	2.30851e+07	0.784451	5643.71	8.59996e+07	14099.1	1.88908e+07	14165.3	1.05021e+07	15684.7	1.46964e+07	70500.2	1423.7	1.25993e+07	17123.7	7247.5	1.67936e+07	45792.5	6996.2	6.30784e+06	46974.8	53423.6	5794.78	1.67936e+07	4408.4	5485.63	1.67936e+07	17022.2	19427.7	15054.6	3992.72	1.67936e+07	14945.9	10275.3	45075.5	837584	1.17997e+06	8.17199e+06	3.10982e+07	nan	nan	nan	nan	nan	nan
78261	20231101.25	28224.5	2.51822e+07	0.81842	5612.19	8.59996e+07	14117.8	1.88908e+07	14186	1.05021e+07	15647.1	1.25993e+07	69365.6	1436.8	1.25993e+07	17440.2	7264.49	1.67936e+07	45894.6	7000.4	6.30784e+06	47638.6	53299.1	5797.39	1.67936e+07	4380.3	5486.18	1.67936e+07	17042	19569.5	17247.3	3994.48	1.67936e+07	14729.7	10279.9	43354.3	835866	1.1715e+06	8.14353e+06	3.07998e+07	nan	nan	nan	nan	nan	nan
78284	20231102.4	27902.1	2.30851e+07	0.775601	5607	8.59996e+07	14039.4	1.88908e+07	14144.7	1.05021e+07	15534.3	1.46964e+07	74128.5	1433.6	1.25993e+07	17090	6870.67	1.88908e+07	43810.2	7007.8	6.30784e+06	47171.4	53039.6	5753.55	1.67936e+07	4380.6	5431.34	1.67936e+07	16902.4	19632.5	14945.6	3982.9	1.67936e+07	14982	10242.7	40950.5	830357	1.18333e+06	8.13473e+06	3.08373e+07	nan	nan	nan	nan	nan	nan
78305	20231102.12	27661.5	2.30851e+07	0.78799	5520.15	8.59996e+07	14034	1.88908e+07	14105	1.05021e+07	15532.4	1.25993e+07	88216.8	1421.1	1.25993e+07	17322.6	6844.95	1.88908e+07	45645.4	7082.4	6.30784e+06	48989.3	53847.3	5797.56	1.67936e+07	4386.2	5725.47	1.88908e+07	17291.3	19701.3	17082.4	4001.42	1.67936e+07	14871.8	10051.9	40964.4	822024	1.17786e+06	8.15592e+06	3.10807e+07	nan	nan	nan	nan	nan	nan
78319	20231102.17	27989.4	2.30851e+07	0.793434	5616.9	8.59996e+07	13956	1.88908e+07	14087.7	1.05021e+07	15500.2	1.46964e+07	83397.2	1432	1.25993e+07	17142.8	6839.82	1.67936e+07	45692.3	6924.1	6.30784e+06	48014.1	53896.9	5801.1	1.67936e+07	4359.7	5739.65	1.67936e+07	17057.3	19245.9	17127.5	3972.71	1.67936e+07	14800.7	10296.3	45008.6	834976	1.18569e+06	8.12273e+06	3.07789e+07	nan	nan	nan	nan	nan	nan
78358	20231103.3	27874.4	2.51822e+07	0.80868	5613.85	8.59996e+07	14024	1.88908e+07	14173.7	1.05021e+07	15658	1.46964e+07	73888.9	1431	1.25993e+07	17311.7	6848.6	1.88908e+07	43799.2	6936	6.30784e+06	48061.9	53988.3	5795.98	1.67936e+07	4395.8	5450.88	1.67936e+07	17328	19780	17379.4	3999.15	1.67936e+07	14648.9	10273.9	45285.7	829705	1.18299e+06	8.14787e+06	3.0569e+07	nan	nan	nan	nan	nan	nan
78365	20231103.6	28034	2.30851e+07	0.787105	5629.97	8.59996e+07	14078.6	1.88908e+07	14064.8	1.05021e+07	15659.9	1.25993e+07	72764.7	1424	1.25993e+07	17440.6	6812	1.67936e+07	45729.3	6988	6.30784e+06	48139.6	54762.3	5761.67	1.67936e+07	4342.9	5755.45	1.67936e+07	17317.7	19576	15085.7	3977.18	1.67936e+07	14683.7	9954.18	43491.4	829913	1.18447e+06	8.14194e+06	3.07286e+07	nan	nan	nan	nan	nan	nan
78372	20231103.8	27795.9	2.30851e+07	0.811727	5578.73	8.59996e+07	13977.9	1.88908e+07	14087	1.05021e+07	15549.8	1.25993e+07	90924.9	1424.7	1.25993e+07	17281.7	6788.55	1.88908e+07	45628	6925.3	6.30784e+06	48368.5	53990.3	5793.13	1.67936e+07	4355.3	5490.55	1.88908e+07	17137.4	19422.4	14922.2	3975.67	1.67936e+07	14674.6	10171.6	44113	838885	1.18174e+06	8.15439e+06	3.16504e+07	nan	nan	nan	nan	nan	nan
78384	20231103.12	27988.7	2.30851e+07	0.802159	5517.44	8.59996e+07	13977	1.88908e+07	14136.3	1.05021e+07	15467.4	1.46964e+07	74382.9	1436.3	1.05021e+07	17108.5	6845.18	1.88908e+07	43878.6	6873	6.30784e+06	47871.9	52246	5794.5	1.67936e+07	4407.6	5481.25	1.67936e+07	17039.1	19823.6	17407.9	3986.29	1.67936e+07	14732.2	10283.4	45856.3	827005	1.1842e+06	8.15429e+06	3.09913e+07	nan	nan	nan	nan	nan	nan
78407	20231103.18	28323.1	2.30851e+07	0.797156	5591.14	8.59996e+07	13982.6	1.88908e+07	14162.3	1.05021e+07	15482.2	1.25993e+07	59381.6	1429.7	1.25993e+07	17161.2	6854.13	1.67936e+07	43521.6	7087.8	6.30784e+06	46519.9	53186.1	5753.69	1.67936e+07	4343	5469.17	1.67936e+07	16972	19625.2	14970	3986.36	1.67936e+07	14529.4	10096.1	40541.7	829355	1.18365e+06	8.17382e+06	3.16191e+07	nan	nan	nan	nan	nan	nan
78430	20231106.1	27904	2.30851e+07	0.785044	5655.91	8.59996e+07	14049.4	1.88908e+07	14103.2	1.05021e+07	15592	1.46964e+07	75314.2	1438.6	1.05021e+07	17142.7	6904.68	1.88908e+07	45699	7080.2	6.30784e+06	47880.3	54141.9	5768.13	1.67936e+07	4390.8	5444.99	1.67936e+07	17165.6	19773.4	15067.1	4003.81	1.67936e+07	14805.8	10259.9	45223.5	830876	1.1802e+06	8.13838e+06	3.08745e+07	nan	nan	nan	nan	nan	nan
78447	20231107.4	27883.9	2.30851e+07	0.803997	5657.58	8.59996e+07	14082.8	1.67936e+07	14226.4	1.05021e+07	15619.5	1.46964e+07	63226	1439.7	1.25993e+07	17233.7	7269.63	1.67936e+07	43850.9	7062.6	6.30784e+06	46545.8	53892.3	5804.83	1.67936e+07	4342.9	5742.73	1.67936e+07	16914.1	19725.8	17400.6	3981.14	1.67936e+07	14863.8	10109.8	43312.1	834561	1.18331e+06	8.16776e+06	3.07268e+07	nan	nan	nan	nan	nan	nan
78486	20231107.16	27770.8	2.51822e+07	0.801874	5617.84	8.59996e+07	14085.9	1.88908e+07	14206.9	1.05021e+07	15610.5	1.46964e+07	87344	1438	1.25993e+07	17279.4	6857.21	1.88908e+07	43772.2	6977.6	6.30784e+06	47733.8	54072.7	5803.25	1.67936e+07	4379	5736.63	1.67936e+07	16987.8	19100.7	15027.3	3971.58	1.67936e+07	14993.4	10303.6	44144	838496	1.18042e+06	8.15371e+06	2.9949e+07	131	1.97092e+06	nan	nan	nan	nan
78494	20231108.1	27924.2	2.51822e+07	0.78468	5532.27	8.59996e+07	13978.1	1.67936e+07	14115.2	1.05021e+07	15527.7	1.25993e+07	80711.6	1433.1	1.25993e+07	17313.3	6857.9	1.88908e+07	45642.2	6971.4	6.30784e+06	47648.1	54269.5	5796.65	1.67936e+07	4387.4	5489.21	1.67936e+07	17062.1	19037.5	15166.2	3996.02	1.67936e+07	15030.2	10352.8	41476.9	804898	1.17648e+06	8.09413e+06	3.07706e+07	124	1.97092e+06	nan	nan	nan	nan
78534	20231108.16	27940.7	2.51822e+07	0.814001	5508.65	8.59996e+07	14045.8	1.67936e+07	14093.1	1.05021e+07	15472.9	1.46964e+07	71362	1417.5	1.25993e+07	17305.2	6760.42	1.67936e+07	43733.1	7034.2	6.30784e+06	47228.6	53728.9	5797.13	1.67936e+07	4372.2	5482.75	1.67936e+07	17222.8	19593.2	14958.9	3994.88	1.67936e+07	14561.2	10207.8	43059	834726	1.17855e+06	8.17421e+06	3.284e+07	127	1.97092e+06	412	1.17179e+07	229	5.94846e+06
78546	20231108.20	27842.3	2.30851e+07	0.803831	5562.49	8.59996e+07	13953.9	1.88908e+07	13971.1	1.05021e+07	15327	1.46964e+07	70955	1401.6	1.25993e+07	17193.3	6825.09	1.88908e+07	45583.3	6884.7	6.30784e+06	47085.4	53757.4	5756.81	1.67936e+07	4369.1	5462.49	1.67936e+07	17214.2	19185.4	17445.6	3947.94	1.67936e+07	14796.9	10162.5	43053.4	834314	1.17795e+06	8.1518e+06	3.07434e+07	132	1.97092e+06	423	1.17179e+07	222	5.94846e+06
78558	20231108.24	27852.3	2.30851e+07	0.791062	5555.81	8.59996e+07	14017.9	1.88908e+07	14164.4	1.05021e+07	15546.1	1.46964e+07	87493	1433.9	1.25993e+07	17037.4	6787.99	1.67936e+07	43865.1	6940.2	6.30784e+06	47170	52802.9	5809.33	1.67936e+07	4353.3	5447.34	1.67936e+07	17050	18865.8	17186.9	3968.11	1.67936e+07	14777.5	10241.2	41786.3	834125	1.18208e+06	8.1247e+06	3.0595e+07	120	1.97092e+06	398	1.17179e+07	224	5.94846e+06
78568	20231109.3	27939.1	2.51822e+07	0.799269	5598.12	8.59996e+07	14087.6	1.88908e+07	14194.3	1.05021e+07	15680	1.46964e+07	87721.9	1434.2	1.25993e+07	17264.6	7226.55	1.67936e+07	43783.2	7100.8	6.30784e+06	47435.4	53481.3	5793.83	1.67936e+07	4385.9	5483.14	1.67936e+07	17091	19520.6	14848.6	3971.48	1.67936e+07	14708.1	10132.6	44947.9	834758	1.18204e+06	8.14421e+06	3.08257e+07	139	1.97092e+06	428	1.17179e+07	238	5.94846e+06

trace_validation_prepare

build_id	build_number	pi_basic_mt_sgx_cft^	pi_basic_mt_sgx_cft_mem	pi_basic_mt_virtual_cft^	Commit latency factor	tpcc_virtual_cft^	ls_virtual_cft^	pi_ls_virtual_cft^	tpcc_sgx_cft^	tpcc_sgx_cft_mem	pi_basic_virtual_cft^	pi_basic_js_virtual_cft^	ls_jwt_virtual_cft^	pi_ls_jwt_virtual_cft^	ls_js_virtual_cft^	ls_sgx_cft^	ls_sgx_cft_mem	pi_ls_sgx_cft^	pi_ls_sgx_cft_mem	ls_full_js_virtual_cft^	pi_basic_sgx_cft^	pi_basic_sgx_cft_mem	ls_js_jwt_virtual_cft^	pi_basic_js_sgx_cft^	pi_basic_js_sgx_cft_mem	ls_jwt_sgx_cft^	ls_jwt_sgx_cft_mem	hist_sgx_cft^	pi_ls_jwt_sgx_cft^	pi_ls_jwt_sgx_cft_mem	ls_js_sgx_cft^	ls_js_sgx_cft_mem	ls_full_js_sgx_cft^	ls_full_js_sgx_cft_mem	ls_js_jwt_sgx_cft^	ls_js_jwt_sgx_cft_mem	tlc_3node_fixed_duration_s	tlc_3node_fixed_states	tlc_atomic_reconfig_duration_s	tlc_atomic_reconfig_states	tlc_reconfig_duration_s	tlc_reconfig_states	RB put (/s)^	CHAMP put (/s)^	RB get (/s)^	CHAMP get (/s)^
78587	20231109.10	28087.7	2.30851e+07	70717.8	0.772136	17272.9	45618.2	47693.3	5580.57	8.59996e+07	53591.1	4381.5	17048	19760.9	17429.7	14056.4	1.88908e+07	14148.2	1.05021e+07	14732.2	15629.8	1.46964e+07	10394.9	1440.8	1.25993e+07	7252.14	1.67936e+07	45826.4	7040.2	6.30784e+06	5775.33	1.67936e+07	5485.43	1.67936e+07	4001.19	1.67936e+07	120	1.97092e+06	403	1.17179e+07	229	5.94846e+06	837616	1.18009e+06	8.15111e+06	3.07785e+07