Skip to content

Commit

Permalink
Increase indexing bulk size from 100 to 5000
Browse files Browse the repository at this point in the history
  • Loading branch information
WilliGautier committed Nov 4, 2024
1 parent 17f6d21 commit 3c3d50a
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 5 deletions.
8 changes: 4 additions & 4 deletions arlas/cli/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ def create(
@indices.command(help="Index data")
def data(
index: str = typer.Argument(help="index's name"),
files: list[str] = typer.Argument(help="List of pathes to the file conaining the data. Format: NDJSON"),
bulk: int = typer.Option(default=100, help="Bulk size for indexing data")
files: list[str] = typer.Argument(help="List of paths to the file(s) containing the data. Format: NDJSON"),
bulk: int = typer.Option(default=5000, help="Bulk size for indexing data")
):
config = variables["arlas"]
i = 1
Expand All @@ -115,9 +115,9 @@ def data(

@indices.command(help="Generate the mapping based on the data")
def mapping(
file: str = typer.Argument(help="Path to the file conaining the data. Format: NDJSON"),
file: str = typer.Argument(help="Path to the file containing the data. Format: NDJSON"),
nb_lines: int = typer.Option(default=2, help="Number of line to consider for generating the mapping. Avoid going over 10."),
field_mapping: list[str] = typer.Option(default=[], help="Overide the mapping with the provided field path/type. Example: fragment.location:geo_point. Important: the full field path must be provided."),
field_mapping: list[str] = typer.Option(default=[], help="Override the mapping with the provided field path/type. Example: fragment.location:geo_point. Important: the full field path must be provided."),
no_fulltext: list[str] = typer.Option(default=[], help="List of keyword or text fields that should not be in the fulltext search. Important: the field name only must be provided."),
push_on: str = typer.Option(default=None, help="Push the generated mapping for the provided index name"),
):
Expand Down
2 changes: 1 addition & 1 deletion arlas/cli/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ def __index_bulk__(arlas: str, index: str, bulk: []):
if result["errors"] is True:
print("ERROR: " + json.dumps(result))

def index_hits(arlas: str, index: str, file_path: str, bulk_size: int = 100, count: int = -1) -> dict[str, int]:
def index_hits(arlas: str, index: str, file_path: str, bulk_size: int = 5000, count: int = -1) -> dict[str, int]:
line_number = 0
line_in_bulk = 0
bulk = []
Expand Down

0 comments on commit 3c3d50a

Please sign in to comment.