Added sign test, updated results,

.gitignore

@@ -11,6 +11,11 @@ plot*
 *lock*
 *.db
 *.ciff
+*.csv
+*.sync*
+*.log
+/trec_eval/
+/try-scipy/
 output*.txt
 results*.txt
 *.txt

compare_phrases_vs_duckdb.py

@@ -0,0 +1,101 @@
+import pandas as pd
+from pathlib import Path
+
+try:
+	from scipy.stats import binomtest
+	HAS_SCIPY = True
+except Exception:
+	HAS_SCIPY = False
+
+
+def main(csv_path: str, out_csv: str = 'comparison_vs_minpmi24.csv'):
+	df = pd.read_csv(csv_path)
+
+
+	df = df.copy()
+	if 'min_freq' in df.columns:
+		df['min_freq'] = df['min_freq']
+	if 'min_pmi' in df.columns:
+		df['min_pmi'] = df['min_pmi']
+	if 'map' in df.columns:
+		df['map'] = df['map']
+
+	group_fields = ['mode', 'stopwords', 'min_freq']
+
+	results = []
+
+	# iterate over groups keyed by (mode, stopwords, min_freq)
+	grouped = df.groupby(group_fields)
+	for key, group in grouped:
+		mode, stopwords, min_freq = key
+
+		# iterate over all min_pmi values present in this group
+		for m in sorted(group['min_pmi'].unique()):
+			# extract series for this min_pmi and baseline rows where min_pmi == 24 (same group)
+			map = pd.to_numeric(group[group['min_pmi'] == m].set_index('query')['map'], errors='coerce')
+
+			# Compare to baseline with min_pmi == 24 and min_freq == 1 (same mode & stopwords)
+			baseline = pd.to_numeric(
+				df[
+					(df['mode'] == mode)
+					& (df['stopwords'] == stopwords)
+					& (df['min_pmi'] == 24)
+					& (df['min_freq'] == 1)
+				].set_index('query')['map'],
+				errors='coerce'
+			)
+
+			# align queries
+			paired = pd.DataFrame({
+				'map': map,
+				'baseline': baseline
+			}).dropna()
+
+			better = int((paired['map'] > paired['baseline']).sum())
+			worse  = int((paired['map'] < paired['baseline']).sum())
+			equal  = int((paired['map'] == paired['baseline']).sum())
+			n_pairs = len(paired)
+			n_sign = better + worse
+
+			p_value = None
+			if HAS_SCIPY and n_sign > 0:
+				# pass number of positives (better) as k to binomtest
+				p_value = float(binomtest(better, n_sign, p=0.5, alternative='two-sided').pvalue)
+
+			results.append({
+				'mode': mode,
+				'stopwords': stopwords,
+				'min_freq': min_freq,
+				'compared_min_pmi': m,
+				'n_pairs': n_pairs,
+				'n_better': better,
+				'n_worse': worse,
+				'n_equal': equal,
+				'p_value': p_value,
+			})
+
+
+	out_df = pd.DataFrame(results)
+	out_df = out_df.sort_values(['mode', 'stopwords', 'min_freq', 'compared_min_pmi'])
+	out_df.to_csv(out_csv, index=False)
+
+	# Print a short summary
+	total_comparisons = len(out_df)
+	print(f"Wrote {out_csv} ({total_comparisons} comparisons)")
+	if total_comparisons > 0:
+		print(out_df.head(20).to_string(index=False))
+
+
+if __name__ == '__main__':
+	CSV = './spreadsheets/results_per_query-1-112.csv'
+	OUT = './spreadsheets/p-values-MAP-q1-112.csv'
+	# CSV = './spreadsheets/results_per_query-113-225.csv'
+	# OUT = 'p-values-MAP-q113-225v2.csv'
+	if not Path(CSV).exists():
+		print(f"Input CSV not found: {CSV}")
+	else:
+		if not HAS_SCIPY:
+			print("scipy not found: binomial p-values will be omitted (set up scipy to get p-values)")
+		main(CSV, OUT)
+
+

compare_postings_cost_vs_duckdb.py

@@ -0,0 +1,97 @@
+import pandas as pd
+from pathlib import Path
+
+try:
+	from scipy.stats import binomtest
+	HAS_SCIPY = True
+except Exception:
+	HAS_SCIPY = False
+
+
+def main(csv_path: str, out_csv: str = 'comparison_vs_minpmi24.csv'):
+	df = pd.read_csv(csv_path)
+	df = df.copy()
+	if 'min_freq' in df.columns:
+		df['min_freq'] = df['min_freq']
+	if 'min_pmi' in df.columns:
+		df['min_pmi'] = df['min_pmi']
+	if 'map' in df.columns:
+		df['map'] = df['map']
+
+	group_fields = ['mode', 'stopwords', 'min_freq']
+
+	results = []
+
+	# iterate over groups keyed by (mode, stopwords, min_freq)
+	grouped = df.groupby(group_fields)
+	for key, group in grouped:
+		mode, stopwords, min_freq = key
+
+		# iterate over all min_pmi values present in this group
+		for m in sorted(group['min_pmi'].unique()):
+			# extract series for this min_pmi and baseline rows where min_pmi == 24 (same group)
+			cip = pd.to_numeric(group[group['min_pmi'] == m].set_index('query')['total_postings_cost'], errors='coerce')
+
+			# Compare to baseline with min_pmi == 24 and min_freq == 1 (same mode & stopwords)
+			baseline = pd.to_numeric(
+				df[
+					(df['mode'] == mode)
+					& (df['stopwords'] == stopwords)
+					& (df['min_pmi'] == 24)
+					& (df['min_freq'] == 1)
+				].set_index('query')['total_postings_cost'],
+				errors='coerce'
+			)
+
+			# align queries
+			paired = pd.DataFrame({
+				'cip': cip,
+				'baseline': baseline
+			}).dropna()
+
+			better = int((paired['cip'] < paired['baseline']).sum())
+			worse  = int((paired['cip'] > paired['baseline']).sum())
+			equal  = int((paired['cip'] == paired['baseline']).sum())
+			n_pairs = len(paired)
+			n_sign = better + worse
+
+			p_value = None
+			if HAS_SCIPY and n_sign > 0:
+				# pass number of positives (better) as k to binomtest
+				p_value = float(binomtest(better, n_sign, p=0.5, alternative='two-sided').pvalue)
+
+			results.append({
+				'mode': mode,
+				'stopwords': stopwords,
+				'min_freq': min_freq,
+				'compared_min_pmi': m,
+				'n_pairs': n_pairs,
+				'n_better': better,
+				'n_worse': worse,
+				'n_equal': equal,
+				'p_value': p_value,
+			})
+
+
+	out_df = pd.DataFrame(results)
+	out_df = out_df.sort_values(['mode', 'stopwords', 'min_freq', 'compared_min_pmi'])
+	out_df.to_csv(out_csv, index=False)
+
+	# Print a short summary
+	total_comparisons = len(out_df)
+	print(f"Wrote {out_csv} ({total_comparisons} comparisons)")
+	if total_comparisons > 0:
+		print(out_df.head(20).to_string(index=False))
+
+
+if __name__ == '__main__':
+	CSV = './spreadsheets/results_per_query-113-225.csv'
+	OUT = './spreadsheets/p-values-CiP-q113-225.csv'
+	if not Path(CSV).exists():
+		print(f"Input CSV not found: {CSV}")
+	else:
+		if not HAS_SCIPY:
+			print("scipy not found: binomial p-values will be omitted (set up scipy to get p-values)")
+		main(CSV, OUT)
+
+

half_split1/1.tsv

@@ -0,0 +1 @@
+1	what similarity laws must be obeyed when constructing aeroelastic models of heated high speed aircraft .

half_split1/10.tsv

@@ -0,0 +1 @@
+10	are real-gas transport properties for air available over a wide range of enthalpies and densities .

half_split1/100.tsv

@@ -0,0 +1 @@
+100	what are the effects of initial imperfections on the elastic buckling of cylindrical shells under axial compression .

half_split1/101.tsv

@@ -0,0 +1 @@
+101	why does the incremental theory and the deformation theory of plastic stress-strain relationship differ greatly when applied to stability problems .

half_split1/102.tsv

@@ -0,0 +1 @@
+102	basic dynamic characteristics of structures continuous over many spans .

half_split1/103.tsv

@@ -0,0 +1 @@
+103	is the information on the buckling of sandwich sphere available .

half_split1/104.tsv

@@ -0,0 +1 @@
+104	can the load deformation characteristics of a beam be obtained with the material being inelastic and a non uniform temperature being present .

half_split1/105.tsv

@@ -0,0 +1 @@
+105	what is the effect of an internal liquid column on the breathing vibrations of a cylindrical shell .

half_split1/106.tsv

@@ -0,0 +1 @@
+106	experimental techniques in shell vibration .

half_split1/107.tsv

@@ -0,0 +1 @@
+107	in summarizing theoretical and experimental work on the behaviour of a typical aircraft structure in a noise environment is it possible to develop a design procedure .

half_split1/108.tsv

@@ -0,0 +1 @@
+108	what data is there on the fatigue of structures under acoustic loading .

half_split1/109.tsv

@@ -0,0 +1 @@
+109	panels subjected to aerodynamic heating .

half_split1/11.tsv

@@ -0,0 +1 @@
+11	is it possible to find an analytical,  similar solution of the strong blast wave problem in the newtonian approximation .

half_split1/110.tsv

@@ -0,0 +1 @@
+110	can increasing the edge loading of a plate beyond the critical value for buckling change the buckling mode .

half_split1/111.tsv

@@ -0,0 +1 @@
+111	have the effects of an elastic edge restraint been considered in previous papers on panel flutter .

half_split1/112.tsv

@@ -0,0 +1 @@
+112	has the solution of the clamped plate problem,  in the classical theory of bending,  been reduced to two successive membrane boundary value problems .

half_split1/12.tsv

@@ -0,0 +1 @@
+12	how can the aerodynamic performance of channel flow ground effect machines be calculated .

half_split1/13.tsv

@@ -0,0 +1 @@
+13	what is the basic mechanism of the transonic aileron buzz .

half_split1/14.tsv

@@ -0,0 +1 @@
+14	papers on shock-sound wave interaction .

half_split1/15.tsv

@@ -0,0 +1 @@
+15	material properties of photoelastic materials .

half_split1/16.tsv

@@ -0,0 +1 @@
+16	can the transverse potential flow about a body of revolution be calculated efficiently by an electronic computer .

half_split1/17.tsv

@@ -0,0 +1 @@
+17	can the three-dimensional problem of a transverse potential flow about a body of revolution be reduced to a two-dimensional problem .

half_split1/18.tsv

@@ -0,0 +1 @@
+18	are experimental pressure distributions on bodies of revolution at angle of attack available .

half_split1/19.tsv

@@ -0,0 +1 @@
+19	does there exist a good basic treatment of the dynamics of re-entry combining consideration of realistic effects with relative simplicity of results .

half_split1/2.tsv

@@ -0,0 +1 @@
+2	what are the structural and aeroelastic problems associated with flight of high speed aircraft .

half_split1/20.tsv

@@ -0,0 +1 @@
+20	has anyone formally determined the influence of joule heating,  produced by the induced current,  in magnetohydrodynamic free convection flows under general conditions .

half_split1/21.tsv

@@ -0,0 +1 @@
+21	why does the compressibility transformation fail to correlate the high speed data for helium and air .

half_split1/22.tsv

@@ -0,0 +1 @@
+22	did anyone else discover that the turbulent skin friction is not over sensitive to the nature of the variation of the viscosity with temperature .

half_split1/23.tsv

@@ -0,0 +1 @@
+23	what progress has been made in research on unsteady aerodynamics .

half_split1/24.tsv

@@ -0,0 +1 @@
+24	what are the factors which influence the time required to invert large structural matrices .

half_split1/25.tsv

@@ -0,0 +1 @@
+25	does a practical flow follow the theoretical concepts for the interaction between adjacent blade rows of a supersonic cascade .

half_split1/26.tsv

@@ -0,0 +1 @@
+26	what is a single approximate formula for the displacement thickness of a laminar boundary layer in compressible flow on a flat plate .

half_split1/27.tsv

@@ -0,0 +1 @@
+27	how is the design of ring or part ring wings by linear theory affected by thickness .

half_split1/28.tsv

@@ -0,0 +1 @@
+28	what application has the linear theory design of curved wings .

half_split1/29.tsv

@@ -0,0 +1 @@
+29	what is the effect of cross sectional shape on the flow over simple delta wings with sharp leading edges .

half_split1/3.tsv

@@ -0,0 +1 @@
+3	what problems of heat conduction in composite slabs have been solved so far .

half_split1/30.tsv

@@ -0,0 +1 @@
+30	papers on flow visualization on slender conical wings .

half_split1/31.tsv

@@ -0,0 +1 @@
+31	what size of end plate can be safely used to simulate two-dimensional flow conditions over a bluff cylindrical body of finite aspect ratio .

half_split1/32.tsv

@@ -0,0 +1 @@
+32	to find an approximate correction for thickness in slender thin-wing theory .

half_split1/33.tsv

@@ -0,0 +1 @@
+33	how do interference-free longitudinal stability measurements (made using free-flight models) compare with similar measurements made in a low-blockage wind tunnel .

half_split1/34.tsv

@@ -0,0 +1 @@
+34	have wind tunnel interference effects been investigated on a systematic basis .

half_split1/35.tsv

@@ -0,0 +1 @@
+35	are there any papers dealing with acoustic wave propagation in reacting gases .

half_split1/36.tsv

@@ -0,0 +1 @@
+36	has anyone investigated relaxation effects on gaseous heat transfer to a suddenly heated wall .

half_split1/37.tsv

@@ -0,0 +1 @@
+37	are there any theoretical methods for predicting base pressure .

half_split1/38.tsv

@@ -0,0 +1 @@
+38	does transition in the hypersonic wake depend on body geometry and size

half_split1/39.tsv

@@ -0,0 +1 @@
+39	how can one detect transition phenomena in boundary layers .

half_split1/4.tsv

@@ -0,0 +1 @@
+4	can a criterion be developed to show empirically the validity of flow solutions for chemically reacting gas mixtures based on the simplifying assumption of instantaneous local chemical equilibrium .

half_split1/40.tsv

@@ -0,0 +1 @@
+40	how can one detect transition phenomena in hypersonic wakes .

half_split1/41.tsv

@@ -0,0 +1 @@
+41	has anyone investigated and developed a simple model for the vortex wake behind a cruciform wing .

half_split1/42.tsv

@@ -0,0 +1 @@
+42	what is a criterion that the transonic flow around an airfoil with a round leading edge be validly analyzed by the linearized transonic flow theory .

half_split1/43.tsv

@@ -0,0 +1 @@
+43	can the transonic flow around an arbitrary smooth thin airfoil be analysed in a simple approximate way .

half_split1/44.tsv

@@ -0,0 +1 @@
+44	what are the details of the rigorous kinetic theory of gases . (chapman-enskog theory) .

half_split1/45.tsv

@@ -0,0 +1 @@
+45	has anyone investigated the effect of surface mass transfer on hypersonic viscous interactions .

half_split1/46.tsv

@@ -0,0 +1 @@
+46	what is the combined effect of surface heat and mass transfer on hypersonic flow .

half_split1/47.tsv

@@ -0,0 +1 @@
+47	what are the existing solutions for hypersonic viscous interactions over an insulated flat plate .

half_split1/48.tsv

@@ -0,0 +1 @@
+48	what controls leading-edge attachment at transonic speeds .

half_split1/49.tsv

@@ -0,0 +1 @@
+49	can the three-point boundary-value problem for the blasius equation be integrated numerically,  using suitable transformations,  without iteration on the boundary conditions .

half_split1/5.tsv

@@ -0,0 +1 @@
+5	what chemical kinetic system is applicable to hypersonic aerodynamic problems .

half_split1/50.tsv

@@ -0,0 +1 @@
+50	what are the effects of small amounts of gas rarefaction on the characteristics of the boundary layers on slender bodies of revolution .

half_split1/51.tsv

@@ -0,0 +1 @@
+51	what is the available information pertaining to boundary layers on very slender bodies of revolution in continuum flow (the ?transverse curvature  effect) .

half_split1/52.tsv

@@ -0,0 +1 @@
+52	what is the available information pertaining to the effect of slight rarefaction on boundary layer flows (the ?slip? effect) .

half_split1/53.tsv

@@ -0,0 +1 @@
+53	what investigations have been made of the flow field about a body moving through a rarefied,  partially ionized gas in the presence of a magnetic field .

half_split1/54.tsv

@@ -0,0 +1 @@
+54	how is the heat transfer downstream of the mass transfer region effected by mass transfer at the nose of a blunted cone .

half_split1/55.tsv

@@ -0,0 +1 @@
+55	to what extent can the available information for incompressible boundary layers be applied to problems involving compressible boundary layers .

half_split1/56.tsv

@@ -0,0 +1 @@
+56	to what extent can readily available steady-state aerodynamic data be utilized to predict lifting-surface flutter characteristics .

half_split1/57.tsv

@@ -0,0 +1 @@
+57	what are the significant steady and non-steady flow characteristics which affect the flutter mechanism .

half_split1/58.tsv

@@ -0,0 +1 @@
+58	is it possible to determine rates of forced convective heat transfer from heated cylinders of non-circular cross-section,  (the fluid flow being along the generators) .

half_split1/59.tsv

@@ -0,0 +1 @@
+59	how much is known about boundary layer flows along non-circular cylinders .

half_split1/6.tsv

@@ -0,0 +1 @@
+6	what theoretical and experimental guides do we have as to turbulent couette flow behaviour .

half_split1/60.tsv

@@ -0,0 +1 @@
+60	is there any simple,  but practical,  method for numerical integration of the mixing problem (i.e. the blasius problem with three-point boundary conditions) .

half_split1/61.tsv

@@ -0,0 +1 @@
+61	does there exist a closed-form expression for the local heat transfer around a yawed cylinder .

half_split1/62.tsv

@@ -0,0 +1 @@
+62	how far around a cylinder and under what conditions of flow,  if any, is the velocity just outside of the boundary layer a linear function of the distance around the cylinder .

half_split1/63.tsv

@@ -0,0 +1 @@
+63	where can i find pressure data on surfaces of swept cylinders .

half_split1/64.tsv

@@ -0,0 +1 @@
+64	can't the static deflection shapes be used in predicting flutter in place of vibrational shapes . if so,  can we provide a justification by means of an example .

half_split1/65.tsv

@@ -0,0 +1 @@
+65	does the boundary layer on a flat plate in a shear flow induce a pressure gradient .

half_split1/66.tsv

@@ -0,0 +1 @@
+66	can the procedure of matching inner and outer solutions for a viscous flow problem be applied when the main stream is a shear flow .

half_split1/67.tsv

@@ -0,0 +1 @@
+67	can series expansions be found for the boundary layer on a flat plate in a shear flow .

half_split1/68.tsv

@@ -0,0 +1 @@
+68	what possible techniques are available for computing the injection distribution corresponding to an isothermal transpiration cooled hemisphere .

half_split1/69.tsv

@@ -0,0 +1 @@
+69	what is known regarding asymptotic solutions to the exact boundary layer equations .

half_split1/7.tsv

@@ -0,0 +1 @@
+7	is it possible to relate the available pressure distributions for an ogive forebody at zero angle of attack to the lower surface pressures of an equivalent ogive forebody at angle of attack .

half_split1/70.tsv

@@ -0,0 +1 @@
+70	previous solutions to the boundary layer similarity equations .

half_split1/71.tsv

@@ -0,0 +1 @@
+71	experimental results on hypersonic viscous interaction .

half_split1/72.tsv

@@ -0,0 +1 @@
+72	what has been done about viscous interactions in relatively low reynolds number flows,  particularly at high mach numbers .

half_split1/73.tsv

@@ -0,0 +1 @@
+73	what role does the effect of chemical reaction (particularly when out of equilibrium) play in the similitude laws governing hypersonic flows over slender aerodynamic bodies .

half_split1/74.tsv

@@ -0,0 +1 @@
+74	how significant is the possible pressure of a dissociated free stream with respect to the realization of hypersonic simulation in high enthalpy wind tunnels .

half_split1/75.tsv

@@ -0,0 +1 @@
+75	do the discrepancies among current analyses of the vorticity effect on stagnation-point heat transfer result primarily from the differences in the viscosity-temperature law assumed .

half_split1/76.tsv

@@ -0,0 +1 @@
+76	how far can one trust the linear viscosity-temperature solution assumed in some of the analyses of hypersonic shock layer at low reynolds number .

half_split1/77.tsv

@@ -0,0 +1 @@
+77	how close is the comparison of the shock layer theory with existing experiments in the low reynolds number (merged-layer) regime .

half_split1/78.tsv

@@ -0,0 +1 @@
+78	has anyone explained the kink in the surge line of a multi-stage axial compressor .

half_split1/79.tsv

@@ -0,0 +1 @@
+79	have any aerodynamic derivatives been measured at hypersonic mach numbers and comparison been made with theoretical work .

half_split1/8.tsv

@@ -0,0 +1 @@
+8	what methods -dash exact or approximate -dash are presently available for predicting body pressures at angle of attack.

half_split1/80.tsv

@@ -0,0 +1 @@
+80	are methods of measuring aerodynamic derivatives available which could be adopted for use in short running time facilities .

half_split1/81.tsv

@@ -0,0 +1 @@
+81	what are wind-tunnel corrections for a two-dimensional aerofoil mounted off-centre in a tunnel .

half_split1/82.tsv

@@ -0,0 +1 @@
+82	how do kuchemann's and multhopp's methods for calculating lift distributions on swept wings in subsonic flow compare with each other and with experiment .

half_split1/83.tsv

@@ -0,0 +1 @@
+83	what is the present state of the theory of quasi-conical flows .

half_split1/84.tsv

@@ -0,0 +1 @@
+84	references on the methods available for accurately estimating aerodynamic heat transfer to conical bodies for both laminar and turbulent flow .

half_split1/85.tsv

@@ -0,0 +1 @@
+85	what parameters can seriously influence natural transition from laminar to turbulent flow on a model in a wind tunnel .