Skip to content

Commit

Permalink
Remove create metadata and ingest range APIs
Browse files Browse the repository at this point in the history
  • Loading branch information
nopcoder committed Sep 18, 2023
1 parent 094e95b commit 26cf3a6
Show file tree
Hide file tree
Showing 57 changed files with 174 additions and 4,028 deletions.
146 changes: 11 additions & 135 deletions api/swagger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ info:
title: lakeFS API
license:
name: "Apache 2.0"
url: "https://www.apache.org/licenses/LICENSE-2.0.html"
version: "0.1.0"
url: https://www.apache.org/licenses/LICENSE-2.0.html
version: 0.1.0

servers:
- url: "/api/v1"
Expand Down Expand Up @@ -151,25 +151,6 @@ components:
minimum: 0
description: Maximal number of entries per page

ImportPagination:
type: object
required:
- has_more
- last_key
properties:
has_more:
type: boolean
description: More keys to be ingested.
continuation_token:
type: string
description: Opaque. Token used to import the next range.
last_key:
type: string
description: Last object store key that was ingested.
staging_token:
type: string
description: Staging token for skipped objects during ingest

Repository:
type: object
required:
Expand Down Expand Up @@ -290,9 +271,9 @@ components:
type: integer
format: int64
description: |
If present and nonzero, physical_address is a presigned URL and
If present and nonzero, physical_address is a pre-signed URL and
will expire at this Unix Epoch time. This will be shorter than
the presigned URL lifetime if an authentication token is about
the pre-signed URL lifetime if an authentication token is about
to expire.
This field is *optional*.
Expand Down Expand Up @@ -1132,14 +1113,14 @@ components:
presigned_url:
type: string
nullable: true
description: if presign=true is passed in the request, this field will contain a presigned URL to use when uploading
description: if presign=true is passed in the request, this field will contain a pre-signed URL to use when uploading
presigned_url_expiry:
type: integer
format: int64
description: |
If present and nonzero, physical_address is a presigned URL and
If present and nonzero, physical_address is a pre-signed URL and
will expire at this Unix Epoch time. This will be shorter than
the presigned URL lifetime if an authentication token is about
the pre-signed URL lifetime if an authentication token is about
to expire.
This field is *optional*.
Expand Down Expand Up @@ -1267,7 +1248,7 @@ components:
description: Path type, can either be 'common_prefix' or 'object'
path:
type: string
description: A source location to ingested path or to a single object. Must match the lakeFS installation blockstore type.
description: A source location to import path or to a single object. Must match the lakeFS installation blockstore type.
example: s3://my-bucket/production/collections/
destination:
type: string
Expand Down Expand Up @@ -1295,32 +1276,6 @@ components:
destination: collections/file1
type: object

StageRangeCreation:
type: object
required:
- fromSourceURI
- after
- prepend
properties:
fromSourceURI:
type: string
description: The source location of the ingested files. Must match the lakeFS installation blockstore type.
example: s3://my-bucket/production/collections/
after:
type: string
description: Only objects after this key would be ingested.
example: production/collections/some/file.parquet
prepend:
type: string
description: A prefix to prepend to ingested objects.
example: collections/
continuation_token:
type: string
description: Opaque. Client should pass the continuation_token received from server to continue creation ranges from the same key.
staging_token:
type: string
description: Opaque. Client should pass staging_token if received from server on previous request

RangeMetadata:
type: object
required:
Expand Down Expand Up @@ -1349,15 +1304,7 @@ components:
type: integer
description: Estimated size of the range in bytes

IngestRangeCreationResponse:
type: object
properties:
range:
$ref: "#/components/schemas/RangeMetadata"
pagination:
$ref: "#/components/schemas/ImportPagination"

ImportStatusResp:
ImportStatus:
type: object
properties:
completed:
Expand Down Expand Up @@ -1889,7 +1836,7 @@ paths:
- $ref: "#/components/parameters/PaginationAmount"
responses:
200:
description: group memeber list
description: group member list
content:
application/json:
schema:
Expand Down Expand Up @@ -3410,7 +3357,7 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/ImportStatusResp"
$ref: "#/components/schemas/ImportStatus"
401:
$ref: "#/components/responses/Unauthorized"
404:
Expand Down Expand Up @@ -3469,77 +3416,6 @@ paths:
default:
$ref: "#/components/responses/ServerError"

/repositories/{repository}/branches/metaranges:
parameters:
- in: path
name: repository
required: true
schema:
type: string
post:
tags:
- import
operationId: createMetaRange
summary: create a lakeFS metarange file from the given ranges
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/MetaRangeCreation"
responses:
201:
description: metarange metadata
content:
application/json:
schema:
$ref: "#/components/schemas/MetaRangeCreationResponse"
400:
$ref: "#/components/responses/ValidationError"
401:
$ref: "#/components/responses/Unauthorized"
403:
$ref: "#/components/responses/Forbidden"
404:
$ref: "#/components/responses/NotFound"
default:
$ref: "#/components/responses/ServerError"

/repositories/{repository}/branches/ranges:
parameters:
- in: path
name: repository
required: true
schema:
type: string
post:
tags:
- import
operationId: ingestRange
summary: create a lakeFS range file from the source uri
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/StageRangeCreation"
responses:
201:
description: range metadata
content:
application/json:
schema:
$ref: "#/components/schemas/IngestRangeCreationResponse"

400:
$ref: "#/components/responses/ValidationError"
401:
$ref: "#/components/responses/Unauthorized"
404:
$ref: "#/components/responses/NotFound"
default:
$ref: "#/components/responses/ServerError"

/repositories/{repository}/branches/{branch}/objects/stage_allowed:
parameters:
- in: path
Expand Down
7 changes: 1 addition & 6 deletions clients/java/README.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 26cf3a6

Please sign in to comment.