Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
G
gogole
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Packages
Packages
Container Registry
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Dos Santos David
gogole
Commits
b72b5236
Commit
b72b5236
authored
Jan 25, 2018
by
Dos Santos David
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
fix ids
parent
c4454c7c
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
10 additions
and
8 deletions
+10
-8
index_command.py
gogole/commands/index_command.py
+3
-3
bsbi_indexer.py
gogole/indexer/bsbi_indexer.py
+1
-1
stanford_parser.py
gogole/parser/stanford_parser.py
+5
-3
main.py
main.py
+1
-1
No files found.
gogole/commands/index_command.py
View file @
b72b5236
...
...
@@ -5,7 +5,7 @@ def run_index_command(collection, args):
indexer
=
collection
.
indexer
if
args
.
index_command
==
'build'
:
build_index
=
Fals
e
build_index
=
args
.
build_no_cach
e
if
not
args
.
build_no_cache
:
# try to load index from the disk
...
...
@@ -18,11 +18,11 @@ def run_index_command(collection, args):
print
(
'... loading the documents...'
)
count_documents
=
0
for
doc
_id
,
document
in
enumerate
(
parser
.
find_documents
(
limit
=
None
)
):
for
doc
ument
in
parser
.
find_documents
(
limit
=
None
):
count_documents
+=
1
tokens
=
tokenizer
.
get_tokens
(
document
)
indexer
.
add_document_tokens
(
doc_id
,
tokens
)
indexer
.
add_document_tokens
(
doc
ument
.
document
_id
,
tokens
)
print
(
'... {} documents loaded'
.
format
(
count_documents
))
...
...
gogole/indexer/bsbi_indexer.py
View file @
b72b5236
...
...
@@ -117,7 +117,7 @@ class BSBIIndexer:
"""
if
token
not
in
self
.
tokens_map
:
return
[]
return
set
()
token_id
=
self
.
tokens_map
[
token
]
...
...
gogole/parser/stanford_parser.py
View file @
b72b5236
from
gogole.document
import
StanfordDocument
from
os
import
listdir
import
itertools
class
StanfordParser
:
...
...
@@ -7,16 +8,17 @@ class StanfordParser:
def
find_documents
(
self
,
limit
=
None
):
counter
=
0
# count documents found
counter
=
itertools
.
count
()
# count documents found
for
collection_index
in
range
(
10
):
collection_dir
=
self
.
DIRECTORY
+
"/"
+
str
(
collection_index
)
for
filename
in
listdir
(
collection_dir
):
with
open
(
collection_dir
+
"/"
+
filename
,
'r'
)
as
f
:
current_document_id
=
str
(
collection_index
)
+
filename
url
=
str
(
collection_index
)
+
filename
current_document_id
=
next
(
counter
)
current_document
=
StanfordDocument
(
current_document_id
)
counter
+=
1
content
=
""
for
line
in
f
:
...
...
main.py
View file @
b72b5236
...
...
@@ -62,7 +62,7 @@ def build_cli_search_parser(root_parser):
search_parser
.
add_argument
(
'--vectorial'
,
action
=
'store_const'
,
const
=
'vectorial'
,
dest
=
'search_query_type'
,
help
=
"use the vectorial model"
)
search_parser
.
add_argument
(
'query'
,
help
=
"your query"
)
search_parser
.
add_argument
(
'query'
,
help
=
"your query"
,
nargs
=
'*'
)
search_parser
.
set_defaults
(
search_query_type
=
'boolean'
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment