Commit cdb05e19 authored by Prot Alexandre's avatar Prot Alexandre

adding precision / recall graphs computing

parent 160e0ef2
......@@ -76,4 +76,4 @@ Sur la collection cacm uniquement, il est possible d'évaluer la pertinence des
gogole > eval <n> # where n is a number of queries to evaluate
```
NB: L'évaluation demande aussi d'avoir construit l'index avec la commande `index build`
\ No newline at end of file
NB: L'évaluation demande aussi d'avoir construit l'index avec la commande `index build` (ou ̀`index build --no-cache`)
......@@ -2,6 +2,7 @@ from gogole import query
from gogole.query import vectorial_query
from gogole.parser import CACMParser
from gogole.parser import QRelsParser
from gogole.utils import heap_law, plot_bar
def run(collection, args):
......@@ -23,6 +24,9 @@ def run(collection, args):
recall_sum = 0
nb_queries = 0
n = 20
recalls_and_precisions_sum = [(0, 0) for _ in range(n)]
# Here a document is a query wrapped in a CACMDocument
for document in cacm_parser.find_documents(limit=nrequests):
q = document.abstract
......@@ -31,7 +35,7 @@ def run(collection, args):
query_browser = query_cls(collection, weight_type)
all_results , t = query_browser.timed_search(q)
n_results = [res for idx, res in enumerate(query_browser.find_n_first_elements(all_results, n=10), start=1)]
n_results = [res for idx, res in enumerate(query_browser.find_n_first_elements(all_results, n), start=1)]
# If there is nothing for this query id, drop it
if document.document_id not in relevant_docs_by_query:
......@@ -41,6 +45,24 @@ def run(collection, args):
intersection_docs = [res for res in n_results if res in relevant_docs]
recalls_and_precisions = [(0, 0) for _ in range(n)]
# Computing the recall / precision values to plot
for k in range(len(n_results)):
k_results = n_results[:(k+1)]
# At rank K
intersection_docs_k = [res for res in k_results if res in relevant_docs]
if len(k_results) > 0:
precision_k = len(intersection_docs_k) / len(k_results)
else:
precision_k = 0
recall_k = len(intersection_docs_k) / len(relevant_docs)
recalls_and_precisions[k] = (recall_k, precision_k)
if len(n_results) != 0:
precision = len(intersection_docs) / len(n_results)
else:
......@@ -55,10 +77,25 @@ def run(collection, args):
avg_mean_precision_sum += avg_mean_precision
nb_queries += 1
for k in range(n):
recall_k, precision_k = recalls_and_precisions[k]
recall_k_sum, precision_k_sum = recalls_and_precisions_sum[k]
recalls_and_precisions_sum[k] = recall_k_sum + recall_k, precision_k_sum + precision_k
precision = precision_sum / nb_queries
recall = recall_sum / nb_queries
avg_mean_precision = avg_mean_precision_sum / nb_queries
# Aggregating the recall/precision values
recalls = []
precisions = []
for k in range(n):
recall_k_sum, precision_k_sum = recalls_and_precisions_sum[k]
recalls.append(recall_k_sum / nb_queries)
precisions.append(precision_k_sum / nb_queries)
print("\n\n{:*^50}\n".format(" Evaluation metrics for weight {weight_type} ".format(weight_type=weight_type)))
print("precision: \t{0:.3g}".format(precision))
print("recall: \t{0:.3g}".format(recall))
......@@ -72,6 +109,45 @@ def run(collection, args):
print("E-Measure: \t{0:.3g}".format(e_measure))
print("F1-Measure: \t{0:.3g}".format(f1_measure))
plot_bar(
recalls,
precisions,
filename='graphs/precision_recall.png',
xlabel="recall",
ylabel="precision"
)
recalls_interpol = []
precisions_interpol = []
best_precision = 0
# Used to plot a point right under another
epsilon = 0.00001
# Interpolation
for k in reversed(range(n)):
recall = recalls[k]
precision = precisions[k]
if k != n-1:
recalls_interpol.append(recall + epsilon)
precisions_interpol.append(best_precision)
if precision > best_precision:
best_precision = precision
recalls_interpol.append(recall)
precisions_interpol.append(best_precision)
plot_bar(
recalls_interpol,
precisions_interpol,
filename='graphs/precision_recall_interpol.png',
xlabel="recall",
ylabel="precision"
)
def compute_avg_mean_precision(doc_ids, relevant_doc_ids):
idx = 0
......
......@@ -61,3 +61,12 @@ Graphes pour la loi de Zipf :
![zipf_law](/graphs/cs276_zipf_law.png)
![zipf_law_logs](/graphs/cs276_zipf_law_logs.png)
### Evaluation
Voici la courbe precision / rappel obtenue sur la collection CACM:
![precision_recall](/graphs/precision_recall_interpol.png)
![precision_recall_logs](/graphs/precision_recall.png)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment