Skip to content

Commit 83a11c6

Browse files
kevjumbakevjumba
authored
fix: Update file api (#2470)
* Update file Signed-off-by: Kevin Zhang <[email protected]> * Fix Signed-off-by: Kevin Zhang <[email protected]> * Fix Signed-off-by: Kevin Zhang <[email protected]> * Lint Signed-off-by: Kevin Zhang <[email protected]> * Update the schema Signed-off-by: Kevin Zhang <[email protected]> * Fix lint Signed-off-by: Kevin Zhang <[email protected]> * Fix lint Signed-off-by: Kevin Zhang <[email protected]> * Fix unit tests Signed-off-by: Kevin Zhang <[email protected]> * Keep file_url param Signed-off-by: Kevin Zhang <[email protected]> * Fix Signed-off-by: Kevin Zhang <[email protected]> * Fix lint Signed-off-by: Kevin Zhang <[email protected]> * Fix Signed-off-by: Kevin Zhang <[email protected]>
1 parent 5be1cc6 commit 83a11c6

File tree

5 files changed

+44
-21
lines changed

5 files changed

+44
-21
lines changed

‎docs/reference/data-sources/file.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ from feast.data_format import ParquetFormat
1616

1717
parquet_file_source = FileSource(
1818
file_format=ParquetFormat(),
19-
file_url="file:///feast/customer.parquet",
19+
path="file:///feast/customer.parquet",
2020
)
2121
```
2222

‎java/serving/src/test/java/feast/serving/util/DataGenerator.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ public static FeatureTableSpec createFeatureTableSpec(
166166
FileFormat.newBuilder()
167167
.setParquetFormat(ParquetFormat.newBuilder().build())
168168
.build())
169-
.setFileUrl("/dev/null")
169+
.setUri("/dev/null")
170170
.build())
171171
.build())
172172
.putAllLabels(labels)
@@ -203,10 +203,7 @@ public static DataSource createFileDataSourceSpec(
203203
return DataSource.newBuilder()
204204
.setType(DataSource.SourceType.BATCH_FILE)
205205
.setFileOptions(
206-
FileOptions.newBuilder()
207-
.setFileFormat(createParquetFormat())
208-
.setFileUrl(fileURL)
209-
.build())
206+
FileOptions.newBuilder().setFileFormat(createParquetFormat()).setUri(fileURL).build())
210207
.setEventTimestampColumn(timestampColumn)
211208
.setDatePartitionColumn(datePartitionColumn)
212209
.build();

‎protos/feast/core/DataSource.proto‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ message DataSource {
8989
// s3://path/to/file for AWS S3 storage
9090
// gs://path/to/file for GCP GCS storage
9191
// file:///path/to/file for local storage
92-
string file_url = 2;
92+
string uri = 2;
9393

9494
// override AWS S3 storage endpoint with custom S3 endpoint
9595
string s3_endpoint_override = 3;

‎sdk/python/feast/infra/offline_stores/file.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,8 @@ def _to_arrow_internal(self):
7777

7878
def persist(self, storage: SavedDatasetStorage):
7979
assert isinstance(storage, SavedDatasetFileStorage)
80-
8180
filesystem, path = FileSource.create_filesystem_and_path(
82-
storage.file_options.file_url, storage.file_options.s3_endpoint_override,
81+
storage.file_options.uri, storage.file_options.s3_endpoint_override,
8382
)
8483

8584
if path.endswith(".parquet"):

‎sdk/python/feast/infra/offline_stores/file_source.py‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import warnings
12
from typing import Callable, Dict, Iterable, Optional, Tuple
23

34
from pyarrow._fs import FileSystem
@@ -61,6 +62,7 @@ def __init__(
6162
self.file_options = FileOptions(
6263
file_format=file_format,
6364
file_url=path,
65+
uri=path,
6466
s3_endpoint_override=s3_endpoint_override,
6567
)
6668

@@ -85,7 +87,6 @@ def __eq__(self, other):
8587

8688
return (
8789
self.name == other.name
88-
and self.file_options.file_url == other.file_options.file_url
8990
and self.file_options.file_format == other.file_options.file_format
9091
and self.event_timestamp_column == other.event_timestamp_column
9192
and self.created_timestamp_column == other.created_timestamp_column
@@ -102,15 +103,15 @@ def path(self):
102103
"""
103104
Returns the path of this file data source.
104105
"""
105-
return self.file_options.file_url
106+
return self.file_options.uri
106107

107108
@staticmethod
108109
def from_proto(data_source: DataSourceProto):
109110
return FileSource(
110111
name=data_source.name,
111112
field_mapping=dict(data_source.field_mapping),
112113
file_format=FileFormat.from_proto(data_source.file_options.file_format),
113-
path=data_source.file_options.file_url,
114+
path=data_source.file_options.uri,
114115
event_timestamp_column=data_source.event_timestamp_column,
115116
created_timestamp_column=data_source.created_timestamp_column,
116117
date_partition_column=data_source.date_partition_column,
@@ -182,17 +183,28 @@ def __init__(
182183
file_format: Optional[FileFormat],
183184
file_url: Optional[str],
184185
s3_endpoint_override: Optional[str],
186+
uri: Optional[str],
185187
):
186188
"""
187189
FileOptions initialization method
188190
189191
Args:
190192
file_format (FileFormat, optional): file source format eg. parquet
191-
file_url (str, optional): file source url eg. s3:// or local file
192-
s3_endpoint_override (str, optional): custom s3 endpoint (used only with s3 file_url)
193+
file_url (str, optional): [DEPRECATED] file source url eg. s3:// or local file
194+
s3_endpoint_override (str, optional): custom s3 endpoint (used only with s3 uri)
195+
uri (str, optional): file source url eg. s3:// or local file
196+
193197
"""
194198
self._file_format = file_format
195-
self._file_url = file_url
199+
if file_url:
200+
warnings.warn(
201+
(
202+
"The option to pass a file_url parameter to FileOptions is being deprecated. "
203+
"Please pass the file url to the uri parameter instead. The parameter will be deprecated in Feast 0.23"
204+
),
205+
DeprecationWarning,
206+
)
207+
self._uri = uri or file_url
196208
self._s3_endpoint_override = s3_endpoint_override
197209

198210
@property
@@ -223,6 +235,20 @@ def file_url(self, file_url):
223235
"""
224236
self._file_url = file_url
225237

238+
@property
239+
def uri(self):
240+
"""
241+
Returns the file url of this file
242+
"""
243+
return self._uri
244+
245+
@uri.setter
246+
def uri(self, uri):
247+
"""
248+
Sets the file url of this file
249+
"""
250+
self._uri = uri
251+
226252
@property
227253
def s3_endpoint_override(self):
228254
"""
@@ -250,7 +276,8 @@ def from_proto(cls, file_options_proto: DataSourceProto.FileOptions):
250276
"""
251277
file_options = cls(
252278
file_format=FileFormat.from_proto(file_options_proto.file_format),
253-
file_url=file_options_proto.file_url,
279+
file_url="",
280+
uri=file_options_proto.uri,
254281
s3_endpoint_override=file_options_proto.s3_endpoint_override,
255282
)
256283
return file_options
@@ -262,12 +289,11 @@ def to_proto(self) -> DataSourceProto.FileOptions:
262289
Returns:
263290
FileOptionsProto protobuf
264291
"""
265-
266292
file_options_proto = DataSourceProto.FileOptions(
267293
file_format=(
268294
None if self.file_format is None else self.file_format.to_proto()
269295
),
270-
file_url=self.file_url,
296+
uri=self.uri,
271297
s3_endpoint_override=self.s3_endpoint_override,
272298
)
273299

@@ -286,16 +312,17 @@ def __init__(
286312
s3_endpoint_override: Optional[str] = None,
287313
):
288314
self.file_options = FileOptions(
289-
file_url=path,
290315
file_format=file_format,
316+
file_url="",
291317
s3_endpoint_override=s3_endpoint_override,
318+
uri=path,
292319
)
293320

294321
@staticmethod
295322
def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage:
296323
file_options = FileOptions.from_proto(storage_proto.file_storage)
297324
return SavedDatasetFileStorage(
298-
path=file_options.file_url,
325+
path=file_options.uri,
299326
file_format=file_options.file_format,
300327
s3_endpoint_override=file_options.s3_endpoint_override,
301328
)
@@ -305,7 +332,7 @@ def to_proto(self) -> SavedDatasetStorageProto:
305332

306333
def to_data_source(self) -> DataSource:
307334
return FileSource(
308-
path=self.file_options.file_url,
335+
path=self.file_options.uri,
309336
file_format=self.file_options.file_format,
310337
s3_endpoint_override=self.file_options.s3_endpoint_override,
311338
)

0 commit comments

Comments
 (0)