Update

dev/requirements.txt

@@ -1,16 +1,12 @@
 # Linter
 mypy==1.8.0
+flake8==3.9.0
 
 # pandas API on Spark Code formatter.
 black==23.9.1
 py
 
-# Spark Connect (required)
-grpcio>=1.62.0
-grpcio-status>=1.62.0
-googleapis-common-protos>=1.56.4
-
-# Spark Connect python proto generation plugin (optional)
+# Spark and Delta Connect python proto generation plugin (optional)
 mypy-protobuf==3.3.0
 googleapis-common-protos-stubs==2.2.0
 grpc-stubs==1.24.11

python/delta/connect/proto/base_pb2.pyi

@@ -31,27 +31,28 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-
 import builtins
 import collections.abc
 import google.protobuf.descriptor
 import google.protobuf.internal.containers
 import google.protobuf.message
-import typing
+import sys
+
+if sys.version_info >= (3, 8):
+    import typing as typing_extensions
+else:
+    import typing_extensions
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing.final
 class DeltaTable(google.protobuf.message.Message):
     """Information required to access a Delta table either by name or by path."""
 
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
-    @typing.final
     class Path(google.protobuf.message.Message):
         DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
-        @typing.final
         class HadoopConfEntry(google.protobuf.message.Message):
             DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -66,7 +67,7 @@ class DeltaTable(google.protobuf.message.Message):
                 value: builtins.str = ...,
             ) -> None: ...
             def ClearField(
-                self, field_name: typing.Literal["key", b"key", "value", b"value"]
+                self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]
             ) -> None: ...
 
         PATH_FIELD_NUMBER: builtins.int
@@ -85,14 +86,15 @@ class DeltaTable(google.protobuf.message.Message):
             hadoop_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
         ) -> None: ...
         def ClearField(
-            self, field_name: typing.Literal["hadoop_conf", b"hadoop_conf", "path", b"path"]
+            self,
+            field_name: typing_extensions.Literal["hadoop_conf", b"hadoop_conf", "path", b"path"],
         ) -> None: ...
 
     PATH_FIELD_NUMBER: builtins.int
     TABLE_OR_VIEW_NAME_FIELD_NUMBER: builtins.int
-    table_or_view_name: builtins.str
     @property
     def path(self) -> global___DeltaTable.Path: ...
+    table_or_view_name: builtins.str
     def __init__(
         self,
         *,
@@ -101,7 +103,7 @@ class DeltaTable(google.protobuf.message.Message):
     ) -> None: ...
     def HasField(
         self,
-        field_name: typing.Literal[
+        field_name: typing_extensions.Literal[
             "access_type",
             b"access_type",
             "path",
@@ -112,7 +114,7 @@ class DeltaTable(google.protobuf.message.Message):
     ) -> builtins.bool: ...
     def ClearField(
         self,
-        field_name: typing.Literal[
+        field_name: typing_extensions.Literal[
             "access_type",
             b"access_type",
             "path",
@@ -122,7 +124,7 @@ class DeltaTable(google.protobuf.message.Message):
         ],
     ) -> None: ...
     def WhichOneof(
-        self, oneof_group: typing.Literal["access_type", b"access_type"]
-    ) -> typing.Literal["path", "table_or_view_name"] | None: ...
+        self, oneof_group: typing_extensions.Literal["access_type", b"access_type"]
+    ) -> typing_extensions.Literal["path", "table_or_view_name"] | None: ...
 
 global___DeltaTable = DeltaTable

python/delta/connect/proto/commands_pb2.pyi

@@ -31,18 +31,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-
 import builtins
 import collections.abc
 import delta.connect.proto.proto.base_pb2
 import google.protobuf.descriptor
 import google.protobuf.internal.containers
 import google.protobuf.message
-import typing
+import sys
+
+if sys.version_info >= (3, 8):
+    import typing as typing_extensions
+else:
+    import typing_extensions
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing.final
 class DeltaCommand(google.protobuf.message.Message):
     """Message to hold all command extensions in Delta Connect."""
 
@@ -58,25 +61,27 @@ class DeltaCommand(google.protobuf.message.Message):
     ) -> None: ...
     def HasField(
         self,
-        field_name: typing.Literal["clone_table", b"clone_table", "command_type", b"command_type"],
+        field_name: typing_extensions.Literal[
+            "clone_table", b"clone_table", "command_type", b"command_type"
+        ],
     ) -> builtins.bool: ...
     def ClearField(
         self,
-        field_name: typing.Literal["clone_table", b"clone_table", "command_type", b"command_type"],
+        field_name: typing_extensions.Literal[
+            "clone_table", b"clone_table", "command_type", b"command_type"
+        ],
     ) -> None: ...
     def WhichOneof(
-        self, oneof_group: typing.Literal["command_type", b"command_type"]
-    ) -> typing.Literal["clone_table"] | None: ...
+        self, oneof_group: typing_extensions.Literal["command_type", b"command_type"]
+    ) -> typing_extensions.Literal["clone_table"] | None: ...
 
 global___DeltaCommand = DeltaCommand
 
-@typing.final
 class CloneTable(google.protobuf.message.Message):
     """Command that creates a copy of a DeltaTable in the specified target location."""
 
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
-    @typing.final
     class PropertiesEntry(google.protobuf.message.Message):
         DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
@@ -91,7 +96,7 @@ class CloneTable(google.protobuf.message.Message):
             value: builtins.str = ...,
         ) -> None: ...
         def ClearField(
-            self, field_name: typing.Literal["key", b"key", "value", b"value"]
+            self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"]
         ) -> None: ...
 
     TABLE_FIELD_NUMBER: builtins.int
@@ -101,6 +106,9 @@ class CloneTable(google.protobuf.message.Message):
     IS_SHALLOW_FIELD_NUMBER: builtins.int
     REPLACE_FIELD_NUMBER: builtins.int
     PROPERTIES_FIELD_NUMBER: builtins.int
+    @property
+    def table(self) -> delta.connect.proto.base_pb2.DeltaTable:
+        """(Required) The source Delta table to clone."""
     target: builtins.str
     """(Required) Path to the location where the cloned table should be stored."""
     version: builtins.int
@@ -112,9 +120,6 @@ class CloneTable(google.protobuf.message.Message):
     replace: builtins.bool
     """(Required) Overwrites the target location when true."""
     @property
-    def table(self) -> delta.connect.proto.base_pb2.DeltaTable:
-        """(Required) The source Delta table to clone."""
-    @property
     def properties(
         self,
     ) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
@@ -134,7 +139,7 @@ class CloneTable(google.protobuf.message.Message):
     ) -> None: ...
     def HasField(
         self,
-        field_name: typing.Literal[
+        field_name: typing_extensions.Literal[
             "table",
             b"table",
             "timestamp",
@@ -147,7 +152,7 @@ class CloneTable(google.protobuf.message.Message):
     ) -> builtins.bool: ...
     def ClearField(
         self,
-        field_name: typing.Literal[
+        field_name: typing_extensions.Literal[
             "is_shallow",
             b"is_shallow",
             "properties",
@@ -167,7 +172,8 @@ class CloneTable(google.protobuf.message.Message):
         ],
     ) -> None: ...
     def WhichOneof(
-        self, oneof_group: typing.Literal["version_or_timestamp", b"version_or_timestamp"]
-    ) -> typing.Literal["version", "timestamp"] | None: ...
+        self,
+        oneof_group: typing_extensions.Literal["version_or_timestamp", b"version_or_timestamp"],
+    ) -> typing_extensions.Literal["version", "timestamp"] | None: ...
 
 global___CloneTable = CloneTable

python/delta/connect/proto/relations_pb2.pyi

@@ -31,16 +31,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-
 import builtins
 import delta.connect.proto.proto.base_pb2
 import google.protobuf.descriptor
 import google.protobuf.message
-import typing
+import sys
+
+if sys.version_info >= (3, 8):
+    import typing as typing_extensions
+else:
+    import typing_extensions
 
 DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
 
-@typing.final
 class Scan(google.protobuf.message.Message):
     """Relation that reads from a Delta table."""
 
@@ -55,7 +58,9 @@ class Scan(google.protobuf.message.Message):
         *,
         table: delta.connect.proto.base_pb2.DeltaTable | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing.Literal["table", b"table"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing.Literal["table", b"table"]) -> None: ...
+    def HasField(
+        self, field_name: typing_extensions.Literal["table", b"table"]
+    ) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["table", b"table"]) -> None: ...
 
 global___Scan = Scan