Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Spark] Add Delta Connect Server Library #3136

Merged
merged 61 commits into from
Jun 3, 2024
Merged
Changes from 1 commit
Commits
Show all changes
61 commits
Select commit Hold shift + click to select a range
c20918d
Add Delta Connect
longvu-db May 21, 2024
9d7f871
Fix merge conflicts
longvu-db May 21, 2024
c1c2f0f
Add generated python files
longvu-db May 21, 2024
08d70fa
Add yaml file
longvu-db May 21, 2024
4956d87
Update
longvu-db May 21, 2024
730d087
add lint ignore
longvu-db May 21, 2024
ba8a4e9
Remove unnecessary Delta repo
longvu-db May 21, 2024
05bcf0c
Rename connect to spark-connect
longvu-db May 21, 2024
7570e84
Update
longvu-db May 22, 2024
5dc923e
Update script
longvu-db May 22, 2024
2364564
Add the necessary mypy ignore stuff
longvu-db May 22, 2024
f4d4538
Remove debug
longvu-db May 22, 2024
84703f2
Remove ,
longvu-db May 22, 2024
1d09b50
Remove unnecessary
longvu-db May 22, 2024
e5f9bca
Undo debug
longvu-db May 22, 2024
a52ec9e
Add Delta Relation
longvu-db May 22, 2024
1095a0e
Try this
longvu-db May 23, 2024
afdb49c
Address comments
longvu-db May 23, 2024
d2b4d1b
Add mypy-protobuf
longvu-db May 24, 2024
d11295b
Swap places
longvu-db May 24, 2024
c3434b8
Try adding buf
longvu-db May 24, 2024
411a5c1
Add python3
longvu-db May 25, 2024
c359273
Address TD's comments
longvu-db May 25, 2024
adbf9ec
Try another way to install buf
longvu-db May 26, 2024
a515f4d
move directory again
longvu-db May 26, 2024
0807676
Try again
longvu-db May 26, 2024
53c1e64
Try again with PATH buf
longvu-db May 27, 2024
8cfc005
Remove Scala tests
longvu-db May 27, 2024
2db4ca9
Add black
longvu-db May 27, 2024
8728810
Disable other checks
longvu-db May 27, 2024
99dc990
Move back
longvu-db May 27, 2024
13a886a
Remove unnecessary comment
longvu-db May 27, 2024
c6a48f3
Add Delta Connect Server Library
longvu-db May 22, 2024
8e71c21
SBT
longvu-db May 23, 2024
44ec48d
Update
longvu-db May 23, 2024
a9f9009
Update
longvu-db May 23, 2024
55d961d
Update
longvu-db May 24, 2024
fbff246
Polish build.sbt
longvu-db May 24, 2024
80f25c6
Move to Spark Master only
longvu-db May 24, 2024
a2776f9
Update
longvu-db May 26, 2024
df0e6fd
Fix build.sbt
longvu-db May 26, 2024
7b3590c
Add back missing files
longvu-db May 26, 2024
215c69a
Fix copyright
longvu-db May 26, 2024
b8f5153
add back delta connect proto
longvu-db May 26, 2024
206d9f7
Update
longvu-db May 26, 2024
5286bbf
Move again
longvu-db May 27, 2024
64dc7b2
Enable again
longvu-db May 27, 2024
137af5f
Update
longvu-db May 27, 2024
e55e837
Remove Clone
longvu-db May 28, 2024
eec6765
Update
longvu-db May 28, 2024
22c5375
Resolve Merge conflicts
longvu-db May 29, 2024
abdba0f
Untab
longvu-db May 29, 2024
e2d5c8f
Resolve Andreas' comments
longvu-db May 29, 2024
e1a4fda
Add publish constraints
longvu-db May 29, 2024
5266b4b
Do not compile, test, or publish Spark Connect common/server projects…
scottsand-db May 29, 2024
f29a7f6
Polish
longvu-db May 30, 2024
9920d94
Nit
longvu-db May 30, 2024
c1c33de
Nit
longvu-db May 30, 2024
1de93c6
Add .
longvu-db May 30, 2024
2bfabcc
Remove dead variable
longvu-db May 30, 2024
5fb9047
Reorganize the imports
longvu-db Jun 3, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update
longvu-db committed May 21, 2024
commit 4956d87f368288695228cd4c847087e8d8a38fd5
8 changes: 2 additions & 6 deletions dev/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
# Linter
mypy==1.8.0
flake8==3.9.0

# pandas API on Spark Code formatter.
black==23.9.1
py

# Spark Connect (required)
grpcio>=1.62.0
grpcio-status>=1.62.0
googleapis-common-protos>=1.56.4

# Spark Connect python proto generation plugin (optional)
# Spark and Delta Connect python proto generation plugin (optional)
mypy-protobuf==3.3.0
googleapis-common-protos-stubs==2.2.0
grpc-stubs==1.24.11
26 changes: 14 additions & 12 deletions python/delta/connect/proto/base_pb2.pyi
Original file line number Diff line number Diff line change
@@ -31,27 +31,28 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import builtins
import collections.abc
import google.protobuf.descriptor
import google.protobuf.internal.containers
import google.protobuf.message
import typing
import sys

if sys.version_info >= (3, 8):
import typing as typing_extensions
else:
import typing_extensions

DESCRIPTOR: google.protobuf.descriptor.FileDescriptor

@typing.final
class DeltaTable(google.protobuf.message.Message):
"""Information required to access a Delta table either by name or by path."""

DESCRIPTOR: google.protobuf.descriptor.Descriptor

@typing.final
class Path(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor

@typing.final
class HadoopConfEntry(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor

@@ -66,7 +67,7 @@ class DeltaTable(google.protobuf.message.Message):
value: builtins.str = ...,
) -> None: ...
def ClearField(
self, field_name: typing.Literal["key", b"key", "value", b"value"]
self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]
) -> None: ...

PATH_FIELD_NUMBER: builtins.int
@@ -85,14 +86,15 @@ class DeltaTable(google.protobuf.message.Message):
hadoop_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing.Literal["hadoop_conf", b"hadoop_conf", "path", b"path"]
self,
field_name: typing_extensions.Literal["hadoop_conf", b"hadoop_conf", "path", b"path"],
) -> None: ...

PATH_FIELD_NUMBER: builtins.int
TABLE_OR_VIEW_NAME_FIELD_NUMBER: builtins.int
table_or_view_name: builtins.str
@property
def path(self) -> global___DeltaTable.Path: ...
table_or_view_name: builtins.str
def __init__(
self,
*,
@@ -101,7 +103,7 @@ class DeltaTable(google.protobuf.message.Message):
) -> None: ...
def HasField(
self,
field_name: typing.Literal[
field_name: typing_extensions.Literal[
"access_type",
b"access_type",
"path",
@@ -112,7 +114,7 @@ class DeltaTable(google.protobuf.message.Message):
) -> builtins.bool: ...
def ClearField(
self,
field_name: typing.Literal[
field_name: typing_extensions.Literal[
"access_type",
b"access_type",
"path",
@@ -122,7 +124,7 @@ class DeltaTable(google.protobuf.message.Message):
],
) -> None: ...
def WhichOneof(
self, oneof_group: typing.Literal["access_type", b"access_type"]
) -> typing.Literal["path", "table_or_view_name"] | None: ...
self, oneof_group: typing_extensions.Literal["access_type", b"access_type"]
) -> typing_extensions.Literal["path", "table_or_view_name"] | None: ...

global___DeltaTable = DeltaTable
40 changes: 23 additions & 17 deletions python/delta/connect/proto/commands_pb2.pyi
Original file line number Diff line number Diff line change
@@ -31,18 +31,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import builtins
import collections.abc
import delta.connect.proto.proto.base_pb2
import google.protobuf.descriptor
import google.protobuf.internal.containers
import google.protobuf.message
import typing
import sys

if sys.version_info >= (3, 8):
import typing as typing_extensions
else:
import typing_extensions

DESCRIPTOR: google.protobuf.descriptor.FileDescriptor

@typing.final
class DeltaCommand(google.protobuf.message.Message):
"""Message to hold all command extensions in Delta Connect."""

@@ -58,25 +61,27 @@ class DeltaCommand(google.protobuf.message.Message):
) -> None: ...
def HasField(
self,
field_name: typing.Literal["clone_table", b"clone_table", "command_type", b"command_type"],
field_name: typing_extensions.Literal[
"clone_table", b"clone_table", "command_type", b"command_type"
],
) -> builtins.bool: ...
def ClearField(
self,
field_name: typing.Literal["clone_table", b"clone_table", "command_type", b"command_type"],
field_name: typing_extensions.Literal[
"clone_table", b"clone_table", "command_type", b"command_type"
],
) -> None: ...
def WhichOneof(
self, oneof_group: typing.Literal["command_type", b"command_type"]
) -> typing.Literal["clone_table"] | None: ...
self, oneof_group: typing_extensions.Literal["command_type", b"command_type"]
) -> typing_extensions.Literal["clone_table"] | None: ...

global___DeltaCommand = DeltaCommand

@typing.final
class CloneTable(google.protobuf.message.Message):
"""Command that creates a copy of a DeltaTable in the specified target location."""

DESCRIPTOR: google.protobuf.descriptor.Descriptor

@typing.final
class PropertiesEntry(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor

@@ -91,7 +96,7 @@ class CloneTable(google.protobuf.message.Message):
value: builtins.str = ...,
) -> None: ...
def ClearField(
self, field_name: typing.Literal["key", b"key", "value", b"value"]
self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]
) -> None: ...

TABLE_FIELD_NUMBER: builtins.int
@@ -101,6 +106,9 @@ class CloneTable(google.protobuf.message.Message):
IS_SHALLOW_FIELD_NUMBER: builtins.int
REPLACE_FIELD_NUMBER: builtins.int
PROPERTIES_FIELD_NUMBER: builtins.int
@property
def table(self) -> delta.connect.proto.base_pb2.DeltaTable:
"""(Required) The source Delta table to clone."""
target: builtins.str
"""(Required) Path to the location where the cloned table should be stored."""
version: builtins.int
@@ -112,9 +120,6 @@ class CloneTable(google.protobuf.message.Message):
replace: builtins.bool
"""(Required) Overwrites the target location when true."""
@property
def table(self) -> delta.connect.proto.base_pb2.DeltaTable:
"""(Required) The source Delta table to clone."""
@property
def properties(
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
@@ -134,7 +139,7 @@ class CloneTable(google.protobuf.message.Message):
) -> None: ...
def HasField(
self,
field_name: typing.Literal[
field_name: typing_extensions.Literal[
"table",
b"table",
"timestamp",
@@ -147,7 +152,7 @@ class CloneTable(google.protobuf.message.Message):
) -> builtins.bool: ...
def ClearField(
self,
field_name: typing.Literal[
field_name: typing_extensions.Literal[
"is_shallow",
b"is_shallow",
"properties",
@@ -167,7 +172,8 @@ class CloneTable(google.protobuf.message.Message):
],
) -> None: ...
def WhichOneof(
self, oneof_group: typing.Literal["version_or_timestamp", b"version_or_timestamp"]
) -> typing.Literal["version", "timestamp"] | None: ...
self,
oneof_group: typing_extensions.Literal["version_or_timestamp", b"version_or_timestamp"],
) -> typing_extensions.Literal["version", "timestamp"] | None: ...

global___CloneTable = CloneTable
15 changes: 10 additions & 5 deletions python/delta/connect/proto/relations_pb2.pyi
Original file line number Diff line number Diff line change
@@ -31,16 +31,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import builtins
import delta.connect.proto.proto.base_pb2
import google.protobuf.descriptor
import google.protobuf.message
import typing
import sys

if sys.version_info >= (3, 8):
import typing as typing_extensions
else:
import typing_extensions

DESCRIPTOR: google.protobuf.descriptor.FileDescriptor

@typing.final
class Scan(google.protobuf.message.Message):
"""Relation that reads from a Delta table."""

@@ -55,7 +58,9 @@ class Scan(google.protobuf.message.Message):
*,
table: delta.connect.proto.base_pb2.DeltaTable | None = ...,
) -> None: ...
def HasField(self, field_name: typing.Literal["table", b"table"]) -> builtins.bool: ...
def ClearField(self, field_name: typing.Literal["table", b"table"]) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["table", b"table"]
) -> builtins.bool: ...
def ClearField(self, field_name: typing_extensions.Literal["table", b"table"]) -> None: ...

global___Scan = Scan
Loading