Commit 5ddcd259 authored by Prot Alexandre's avatar Prot Alexandre

adding evaluation metrics

parent 4f6a914f
...@@ -72,6 +72,8 @@ gogole > search -v --norm-freq <query> # fréquences normalisées ...@@ -72,6 +72,8 @@ gogole > search -v --norm-freq <query> # fréquences normalisées
Sur la collection cacm uniquement, il est possible d'évaluer la pertinence des recherches vectorielles effectuées avec les différentes pondérations. Sur la collection cacm uniquement, il est possible d'évaluer la pertinence des recherches vectorielles effectuées avec les différentes pondérations.
```python
gogole > eval <n> # where n is a number of queries to evaluate
```
NB: L'évaluation demande aussi d'avoir construit l'index avec la commande `index build` NB: L'évaluation demande aussi d'avoir construit l'index avec la commande `index build`
\ No newline at end of file
...@@ -3,17 +3,23 @@ from gogole.query import vectorial_query ...@@ -3,17 +3,23 @@ from gogole.query import vectorial_query
from gogole.parser import CACMParser from gogole.parser import CACMParser
from gogole.parser import QRelsParser from gogole.parser import QRelsParser
def run(collection, args): def run(collection, args):
# Runs the CACM Parser on the queries file with the same structure # Runs the CACM Parser on the queries file with the same structure
cacm_parser = CACMParser("data/query.text") cacm_parser = CACMParser("data/query.text")
nrequests = int(args.nrequests[0]) nrequests = int(args.nrequests[0])
if nrequests < 1:
print("Please enter a non-zero number of test queries")
return
qrels_parser = QRelsParser() qrels_parser = QRelsParser()
relevant_docs_by_query = qrels_parser.parse_all(nrequests) relevant_docs_by_query = qrels_parser.parse_all(nrequests)
for weight_type in vectorial_query.WEIGHTING_TYPES: for weight_type in vectorial_query.WEIGHTING_TYPES:
precision_sum = 0 precision_sum = 0
avg_mean_precision_sum = 0
recall_sum = 0 recall_sum = 0
nb_queries = 0 nb_queries = 0
...@@ -26,11 +32,11 @@ def run(collection, args): ...@@ -26,11 +32,11 @@ def run(collection, args):
all_results , t = query_browser.timed_search(q) all_results , t = query_browser.timed_search(q)
n_results = [res for idx, res in enumerate(query_browser.find_n_first_elements(all_results, n=10), start=1)] n_results = [res for idx, res in enumerate(query_browser.find_n_first_elements(all_results, n=10), start=1)]
# If there is nothing for this query id, drop it # If there is nothing for this query id, drop it
if document.document_id not in relevant_docs_by_query: if document.document_id not in relevant_docs_by_query:
continue continue
relevant_docs = relevant_docs_by_query[document.document_id] relevant_docs = relevant_docs_by_query[document.document_id]
intersection_docs = [res for res in n_results if res in relevant_docs] intersection_docs = [res for res in n_results if res in relevant_docs]
...@@ -41,11 +47,43 @@ def run(collection, args): ...@@ -41,11 +47,43 @@ def run(collection, args):
precision = 0 precision = 0
recall = len(intersection_docs) / len(relevant_docs) recall = len(intersection_docs) / len(relevant_docs)
avg_mean_precision = compute_avg_mean_precision(n_results, relevant_docs)
precision_sum += precision precision_sum += precision
recall_sum += recall recall_sum += recall
avg_mean_precision_sum += avg_mean_precision
nb_queries += 1 nb_queries += 1
precision = precision_sum / nb_queries precision = precision_sum / nb_queries
recall = recall_sum / nb_queries recall = recall_sum / nb_queries
print("for weight {weight}: precision: {precision}, rappel: {recall}".format(weight=weight_type, precision=precision, recall=recall)) avg_mean_precision = avg_mean_precision_sum / nb_queries
print("\n\n{:*^50}\n".format(" Evaluation metrics for weight {weight_type} ".format(weight_type=weight_type)))
print("precision: \t{0:.3g}".format(precision))
print("recall: \t{0:.3g}".format(recall))
print("MA Precision: \t{0:.3g}".format(avg_mean_precision))
if precision * recall != 0:
beta = precision / recall
e_measure = 1 - ((beta ** 2) + 1) * precision * recall / ((beta ** 2) * precision + recall)
f_measure = 1 - e_measure
f1_measure = (2 * precision * recall) / (precision + recall)
print("E-Measure: \t{0:.3g}".format(e_measure))
print("F1-Measure: \t{0:.3g}".format(f1_measure))
def compute_avg_mean_precision(doc_ids, relevant_doc_ids):
idx = 0
nb_good_results = 0
avg_mean_precision = 0
for res in doc_ids:
idx += 1
if res in relevant_doc_ids:
nb_good_results += 1
avg_mean_precision += nb_good_results / idx
if nb_good_results > 0:
avg_mean_precision = avg_mean_precision / nb_good_results
return avg_mean_precision
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment