MS MARCO V1 Passage

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-default.dl20.txt \
  --bm25 --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-default.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-default.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-default.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-default.dev.txt \
  --bm25 --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-default.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-default.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rm3-default.dl20.txt \
  --bm25 --k1 0.9 --b 0.4 --rm3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-default.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-default.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-default.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rm3-default.dev.txt \
  --bm25 --k1 0.9 --b 0.4 --rm3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-default.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-default.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rocchio-default.dl20.txt \
  --bm25 --k1 0.9 --b 0.4 --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-default.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-default.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-default.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rocchio-default.dev.txt \
  --bm25 --k1 0.9 --b 0.4 --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-default.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-default.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --topics dl20 \
  --index msmarco-v1-passage \
  --output run.msmarco-v1-passage.bm25-tuned.dl20.txt \
  --bm25

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-tuned.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --topics msmarco-passage-dev-subset \
  --index msmarco-v1-passage \
  --output run.msmarco-v1-passage.bm25-tuned.dev.txt \
  --bm25

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-tuned.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-tuned.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rm3-tuned.dl20.txt \
  --bm25 --rm3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-tuned.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rm3-tuned.dev.txt \
  --bm25 --rm3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-tuned.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-tuned.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rocchio-tuned.dl20.txt \
  --bm25 --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-tuned.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rocchio-tuned.dev.txt \
  --bm25 --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-tuned.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-tuned.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5 \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-d2q-t5-default.dl20.txt \
  --bm25 --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-d2q-t5-default.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-d2q-t5-default.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-d2q-t5-default.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5 \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-d2q-t5-default.dev.txt \
  --bm25 --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-d2q-t5-default.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-d2q-t5-default.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dl20.txt \
  --bm25 --rm3 --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dev.txt \
  --bm25 --rm3 --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-default.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dl20.txt \
  --bm25 --rocchio --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dev.txt \
  --bm25 --rocchio --k1 0.9 --b 0.4

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-default.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5 \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-d2q-t5-tuned.dl20.txt \
  --bm25

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-d2q-t5-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-d2q-t5-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-d2q-t5-tuned.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5 \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-d2q-t5-tuned.dev.txt \
  --bm25

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-d2q-t5-tuned.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-d2q-t5-tuned.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dl20.txt \
  --bm25 --rm3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dev.txt \
  --bm25 --rm3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rm3-d2q-t5-tuned.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics dl20 \
  --output run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dl20.txt \
  --bm25 --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-d2q-t5-docvectors \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dev.txt \
  --bm25 --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bm25-rocchio-d2q-t5-tuned.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil \
  --topics dl20-unicoil \
  --output run.msmarco-v1-passage.unicoil.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.unicoil.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.unicoil.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.unicoil.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil \
  --topics msmarco-passage-dev-subset-unicoil \
  --output run.msmarco-v1-passage.unicoil.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil \
  --topics dl20 \
  --encoder castorini/unicoil-msmarco-passage \
  --output run.msmarco-v1-passage.unicoil-pytorch.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.unicoil-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.unicoil-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.unicoil-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/unicoil-msmarco-passage \
  --output run.msmarco-v1-passage.unicoil-pytorch.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil \
  --topics dl20 \
  --onnx-encoder UniCoil \
  --output run.msmarco-v1-passage.unicoil-onnx.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.unicoil-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.unicoil-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.unicoil-onnx.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil \
  --topics msmarco-passage-dev-subset \
  --onnx-encoder UniCoil \
  --output run.msmarco-v1-passage.unicoil-onnx.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-onnx.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-onnx.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil-noexp \
  --topics dl20-unicoil-noexp \
  --output run.msmarco-v1-passage.unicoil-noexp.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil-noexp \
  --topics msmarco-passage-dev-subset-unicoil-noexp \
  --output run.msmarco-v1-passage.unicoil-noexp.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-noexp.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-noexp.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil-noexp \
  --topics dl20 \
  --encoder castorini/unicoil-noexp-msmarco-passage \
  --output run.msmarco-v1-passage.unicoil-noexp-pytorch.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil-noexp \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/unicoil-noexp-msmarco-passage \
  --output run.msmarco-v1-passage.unicoil-noexp-pytorch.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-noexp-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-noexp-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil-noexp \
  --topics dl20 \
  --onnx-encoder UniCoil \
  --output run.msmarco-v1-passage.unicoil-noexp-onnx.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.unicoil-noexp-onnx.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-unicoil-noexp \
  --topics msmarco-passage-dev-subset \
  --onnx-encoder UniCoil \
  --output run.msmarco-v1-passage.unicoil-noexp-onnx.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-noexp-onnx.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.unicoil-noexp-onnx.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed \
  --topics dl20 \
  --encoder naver/splade-cocondenser-ensembledistil \
  --output run.msmarco-v1-passage.splade-pp-ed-pytorch.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed \
  --topics msmarco-passage-dev-subset \
  --encoder naver/splade-cocondenser-ensembledistil \
  --output run.msmarco-v1-passage.splade-pp-ed-pytorch.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed \
  --topics dl20 \
  --onnx-encoder SpladePlusPlusEnsembleDistil \
  --output run.msmarco-v1-passage.splade-pp-ed-onnx.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-onnx.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed \
  --topics msmarco-passage-dev-subset \
  --onnx-encoder SpladePlusPlusEnsembleDistil \
  --output run.msmarco-v1-passage.splade-pp-ed-onnx.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-onnx.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-onnx.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed-text \
  --topics dl20 \
  --encoder naver/splade-cocondenser-ensembledistil \
  --output run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dl20.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed-text \
  --topics msmarco-passage-dev-subset \
  --encoder naver/splade-cocondenser-ensembledistil \
  --output run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dev.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed-text \
  --topics dl20 \
  --onnx-encoder SpladePlusPlusEnsembleDistil \
  --output run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dl20.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-ed-text \
  --topics msmarco-passage-dev-subset \
  --onnx-encoder SpladePlusPlusEnsembleDistil \
  --output run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dev.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-ed-rocchio-onnx.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd \
  --topics dl20 \
  --encoder naver/splade-cocondenser-selfdistil \
  --output run.msmarco-v1-passage.splade-pp-sd-pytorch.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd \
  --topics msmarco-passage-dev-subset \
  --encoder naver/splade-cocondenser-selfdistil \
  --output run.msmarco-v1-passage.splade-pp-sd-pytorch.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd \
  --topics dl20 \
  --onnx-encoder SpladePlusPlusSelfDistil \
  --output run.msmarco-v1-passage.splade-pp-sd-onnx.dl20.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-onnx.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd \
  --topics msmarco-passage-dev-subset \
  --onnx-encoder SpladePlusPlusSelfDistil \
  --output run.msmarco-v1-passage.splade-pp-sd-onnx.dev.txt \
  --hits 1000 --impact

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-onnx.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-onnx.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd-text \
  --topics dl20 \
  --encoder naver/splade-cocondenser-selfdistil \
  --output run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dl20.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd-text \
  --topics msmarco-passage-dev-subset \
  --encoder naver/splade-cocondenser-selfdistil \
  --output run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dev.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd-text \
  --topics dl20 \
  --onnx-encoder SpladePlusPlusSelfDistil \
  --output run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dl20.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-splade-pp-sd-text \
  --topics msmarco-passage-dev-subset \
  --onnx-encoder SpladePlusPlusSelfDistil \
  --output run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dev.txt \
  --hits 1000 --impact --rocchio

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.splade-pp-sd-rocchio-onnx.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics dl20 --encoded-queries ance-dl20 \
  --output run.msmarco-v1-passage.ance.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.ance.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.ance.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.ance.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics msmarco-passage-dev-subset --encoded-queries ance-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.ance.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics dl20 \
  --encoder castorini/ance-msmarco-passage \
  --output run.msmarco-v1-passage.ance-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.ance-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.ance-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.ance-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/ance-msmarco-passage \
  --output run.msmarco-v1-passage.ance-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics dl20 \
  --encoder castorini/ance-msmarco-passage \
  --output run.msmarco-v1-passage.ance-avg-prf-pytorch.dl20.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.ance-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.ance-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.ance-avg-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/ance-msmarco-passage \
  --output run.msmarco-v1-passage.ance-avg-prf-pytorch.dev.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance-avg-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance-avg-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics dl20 \
  --encoder castorini/ance-msmarco-passage \
  --output run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dl20.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.ance \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/ance-msmarco-passage \
  --output run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dev.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.ance-rocchio-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.sbert \
  --topics dl20 \
  --encoder sentence-transformers/msmarco-distilbert-base-v3 \
  --output run.msmarco-v1-passage.sbert-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.sbert-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.sbert-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.sbert-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.sbert \
  --topics msmarco-passage-dev-subset \
  --encoder sentence-transformers/msmarco-distilbert-base-v3 \
  --output run.msmarco-v1-passage.sbert-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.sbert-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.sbert-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.sbert \
  --topics dl20 \
  --encoder sentence-transformers/msmarco-distilbert-base-v3 \
  --output run.msmarco-v1-passage.sbert-avg-prf-pytorch.dl20.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.sbert-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.sbert-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.sbert-avg-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.sbert \
  --topics msmarco-passage-dev-subset \
  --encoder sentence-transformers/msmarco-distilbert-base-v3 \
  --output run.msmarco-v1-passage.sbert-avg-prf-pytorch.dev.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.sbert-avg-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.sbert-avg-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.sbert \
  --topics dl20 \
  --encoder sentence-transformers/msmarco-distilbert-base-v3 \
  --output run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dl20.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.sbert \
  --topics msmarco-passage-dev-subset \
  --encoder sentence-transformers/msmarco-distilbert-base-v3 \
  --output run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dev.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.sbert-rocchio-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-margin-mse-t2 \
  --topics dl20 --encoded-queries distilbert_kd-dl20 \
  --output run.msmarco-v1-passage.distilbert-kd.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.distilbert-kd.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-margin-mse-t2 \
  --topics msmarco-passage-dev-subset --encoded-queries distilbert_kd-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.distilbert-kd.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-margin-mse-t2 \
  --topics dl20 \
  --encoder sebastian-hofstaetter/distilbert-dot-margin_mse-T2-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-margin-mse-t2 \
  --topics msmarco-passage-dev-subset \
  --encoder sebastian-hofstaetter/distilbert-dot-margin_mse-T2-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics dl20 --encoded-queries distilbert_tas_b-dl20 \
  --output run.msmarco-v1-passage.distilbert-kd-tasb.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics msmarco-passage-dev-subset --encoded-queries distilbert_tas_b-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.distilbert-kd-tasb.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics dl20 \
  --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics msmarco-passage-dev-subset \
  --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics dl20 \
  --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dl20.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics msmarco-passage-dev-subset \
  --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dev.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb-avg-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics dl20 \
  --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dl20.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.distilbert-dot-tas_b-b256 \
  --topics msmarco-passage-dev-subset \
  --encoder sebastian-hofstaetter/distilbert-dot-tas_b-b256-msmarco \
  --output run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dev.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.distilbert-kd-tasb-rocchio-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics dl20 --encoded-queries tct_colbert-v2-hnp-dl20 \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics msmarco-passage-dev-subset --encoded-queries tct_colbert-v2-hnp-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics dl20 \
  --encoder castorini/tct_colbert-v2-hnp-msmarco \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/tct_colbert-v2-hnp-msmarco \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics dl20 \
  --encoder castorini/tct_colbert-v2-hnp-msmarco \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dl20.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/tct_colbert-v2-hnp-msmarco \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dev.txt \
  --prf-method avg --prf-depth 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-avg-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics dl20 \
  --encoder castorini/tct_colbert-v2-hnp-msmarco \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dl20.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.tct_colbert-v2-hnp \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/tct_colbert-v2-hnp-msmarco \
  --output run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dev.txt \
  --prf-method rocchio --prf-depth 5 --rocchio-alpha 0.4 --rocchio-beta 0.6 --rocchio-topk 5

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-rocchio-prf-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.hybrid \
  dense  --index msmarco-v1-passage.tct_colbert-v2-hnp \
         --encoder castorini/tct_colbert-v2-hnp-msmarco \
  sparse --index msmarco-v1-passage \
  fusion --alpha 0.06 \
  run    --threads 16 --batch-size 512 \
         --topics dl20 \
         --output run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.hybrid \
  dense  --index msmarco-v1-passage.tct_colbert-v2-hnp \
         --encoder castorini/tct_colbert-v2-hnp-msmarco \
  sparse --index msmarco-v1-passage \
  fusion --alpha 0.06 \
  run    --threads 16 --batch-size 512 \
         --topics msmarco-passage-dev-subset \
         --output run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.hybrid \
  dense  --index msmarco-v1-passage.tct_colbert-v2-hnp \
         --encoder castorini/tct_colbert-v2-hnp-msmarco \
  sparse --index msmarco-v1-passage-d2q-t5 \
  fusion --alpha 0.1 \
  run    --threads 16 --batch-size 512 \
         --topics dl20 \
         --output run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.hybrid \
  dense  --index msmarco-v1-passage.tct_colbert-v2-hnp \
         --encoder castorini/tct_colbert-v2-hnp-msmarco \
  sparse --index msmarco-v1-passage-d2q-t5 \
  fusion --alpha 0.1 \
  run    --threads 16 --batch-size 512 \
         --topics msmarco-passage-dev-subset \
         --output run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.tct_colbert-v2-hnp-bm25d2q-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-slimr \
  --topics dl20 \
  --encoder castorini/slimr-msmarco-passage \
  --encoded-corpus scipy-sparse-vectors.msmarco-v1-passage-slimr \
  --output run.msmarco-v1-passage.slimr.dl20.txt \
  --output-format msmarco --hits 1000 --impact --min-idf 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.slimr.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.slimr.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.slimr.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-slimr \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/slimr-msmarco-passage \
  --encoded-corpus scipy-sparse-vectors.msmarco-v1-passage-slimr \
  --output run.msmarco-v1-passage.slimr.dev.txt \
  --output-format msmarco --hits 1000 --impact --min-idf 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.slimr.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.slimr.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-slimr-pp \
  --topics dl20 \
  --encoder castorini/slimr-pp-msmarco-passage \
  --encoded-corpus scipy-sparse-vectors.msmarco-v1-passage-slimr-pp \
  --output run.msmarco-v1-passage.slimr-pp.dl20.txt \
  --output-format msmarco --hits 1000 --impact --min-idf 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.slimr-pp.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.slimr-pp.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.slimr-pp.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.lucene \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage-slimr-pp \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/slimr-pp-msmarco-passage \
  --encoded-corpus scipy-sparse-vectors.msmarco-v1-passage-slimr-pp \
  --output run.msmarco-v1-passage.slimr-pp.dev.txt \
  --output-format msmarco --hits 1000 --impact --min-idf 3

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.slimr-pp.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.slimr-pp.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.aggretriever-distilbert \
  --topics dl20 \
  --encoder castorini/aggretriever-distilbert \
  --output run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.aggretriever-distilbert \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/aggretriever-distilbert \
  --output run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.aggretriever-distilbert-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.aggretriever-cocondenser \
  --topics dl20 \
  --encoder castorini/aggretriever-cocondenser \
  --output run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.aggretriever-cocondenser \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/aggretriever-cocondenser \
  --output run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.aggretriever-cocondenser-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage.openai-ada2 \
  --topics dl20 --encoded-queries openai-ada2-dl20 \
  --output run.msmarco-v1-passage.openai-ada2.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.openai-ada2.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.openai-ada2.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.openai-ada2.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage.openai-ada2 \
  --topics msmarco-passage-dev-subset --encoded-queries openai-ada2-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.openai-ada2.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.openai-ada2.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.openai-ada2.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 128 \
  --index msmarco-v1-passage.openai-ada2 \
  --topics dl20 --encoded-queries openai-ada2-dl20-hyde \
  --output run.msmarco-v1-passage.openai-ada2-hyde.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.openai-ada2-hyde.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.openai-ada2-hyde.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.openai-ada2-hyde.dl20.txt

Not available.

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.openai-text-embedding-3-large \
  --topics dl20 --encoded-queries openai-text-embedding-3-large-dl20 \
  --output run.msmarco-v1-passage.openai-text-embedding-3-large.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.openai-text-embedding-3-large.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.openai-text-embedding-3-large.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.openai-text-embedding-3-large.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.openai-text-embedding-3-large \
  --topics msmarco-passage-dev-subset --encoded-queries openai-text-embedding-3-large-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.openai-text-embedding-3-large.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.openai-text-embedding-3-large.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.openai-text-embedding-3-large.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.cosdpr-distil  \
  --topics dl20 \
  --encoder castorini/cosdpr-distil \
  --output run.msmarco-v1-passage.cosdpr-distil-pytorch.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.cosdpr-distil-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.cosdpr-distil-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.cosdpr-distil-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.cosdpr-distil  \
  --topics msmarco-passage-dev-subset \
  --encoder castorini/cosdpr-distil \
  --output run.msmarco-v1-passage.cosdpr-distil-pytorch.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.cosdpr-distil-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.cosdpr-distil-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --encoder-class auto \
  --encoder BAAI/bge-base-en-v1.5 --l2-norm --query-prefix "Represent this sentence for searching relevant passages:" \
  --index msmarco-v1-passage.bge-base-en-v1.5 \
  --topics dl20 \
  --output run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dl20.txt \
  --hits 1000

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --encoder-class auto \
  --encoder BAAI/bge-base-en-v1.5 --l2-norm --query-prefix "Represent this sentence for searching relevant passages:" \
  --index msmarco-v1-passage.bge-base-en-v1.5 \
  --topics msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dev.txt \
  --hits 1000

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.bge-base-en-v1.5-pytorch.dev.txt

Command to generate run on TREC 2020 queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.cohere-embed-english-v3.0 \
  --topics dl20 --encoded-queries cohere-embed-english-v3.0-dl20 \
  --output run.msmarco-v1-passage.cohere-embed-english-v3.0.dl20.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -l 2 -m map dl20-passage \
  run.msmarco-v1-passage.cohere-embed-english-v3.0.dl20.txt
python -m pyserini.eval.trec_eval -c -m ndcg_cut.10 dl20-passage \
  run.msmarco-v1-passage.cohere-embed-english-v3.0.dl20.txt
python -m pyserini.eval.trec_eval -c -l 2 -m recall.1000 dl20-passage \
  run.msmarco-v1-passage.cohere-embed-english-v3.0.dl20.txt

Command to generate run on dev queries:

python -m pyserini.search.faiss \
  --threads 16 --batch-size 512 \
  --index msmarco-v1-passage.cohere-embed-english-v3.0 \
  --topics msmarco-passage-dev-subset --encoded-queries cohere-embed-english-v3.0-msmarco-passage-dev-subset \
  --output run.msmarco-v1-passage.cohere-embed-english-v3.0.dev.txt

Evaluation commands:

python -m pyserini.eval.trec_eval -c -M 10 -m recip_rank msmarco-passage-dev-subset \
  run.msmarco-v1-passage.cohere-embed-english-v3.0.dev.txt
python -m pyserini.eval.trec_eval -c -m recall.1000 msmarco-passage-dev-subset \
  run.msmarco-v1-passage.cohere-embed-english-v3.0.dev.txt