Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: MobileTeleSystems/onetl
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 7cdae8a46009faf657b3d70f165e93de14857f5d
Choose a base ref
..
head repository: MobileTeleSystems/onetl
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: dcacfae134b560ee0f8e98a8d010c3b2a82640fb
Choose a head ref
2 changes: 1 addition & 1 deletion .github/workflows/data/core/matrix.yml
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ min: &min
max: &max
spark-version: 3.5.1
pydantic-version: 2
python-version: '3.13.0-beta.4'
python-version: '3.13.0-beta.4-dev'
java-version: 20
os: ubuntu-latest

2 changes: 1 addition & 1 deletion .github/workflows/test-sftp.yml
Original file line number Diff line number Diff line change
@@ -58,7 +58,7 @@ jobs:

- name: Install dependencies
run: |
pip install -I -r requirements/core.txt -r requirements/sftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt
pip install -I -r requirements/core.txt -r requirements/sftp.txt -r requirements/tests/base.txt -r requirements/tests/pydantic-${{ inputs.pydantic-version }}.txt cffi==1.17.0rc1
- name: Run tests
run: |
6 changes: 6 additions & 0 deletions .github/workflows/test-webdav.yml
Original file line number Diff line number Diff line change
@@ -33,6 +33,12 @@ jobs:
with:
python-version: ${{ inputs.python-version }}

- name: Set up lxml libs
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install --no-install-recommends libxml2-dev libxslt-dev
- name: Cache pip
uses: actions/cache@v4
if: inputs.with-cache
2 changes: 1 addition & 1 deletion onetl/connection/db_connection/greenplum/connection.py
Original file line number Diff line number Diff line change
@@ -76,7 +76,7 @@ class Config:


@support_hooks
class Greenplum(JDBCMixin, DBConnection):
class Greenplum(JDBCMixin, DBConnection): # noqa: WPS338
"""Greenplum connection. |support_hooks|
Based on package ``io.pivotal:greenplum-spark:2.2.0``
23 changes: 20 additions & 3 deletions onetl/connection/db_connection/jdbc_connection/connection.py
Original file line number Diff line number Diff line change
@@ -9,10 +9,11 @@
from typing import TYPE_CHECKING, Any, ClassVar, Optional

try:
from pydantic.v1 import PrivateAttr, SecretStr
from pydantic.v1 import PrivateAttr, SecretStr, validator
except (ImportError, AttributeError):
from pydantic import PrivateAttr, SecretStr # type: ignore[no-redef, assignment]
from pydantic import PrivateAttr, SecretStr, validator # type: ignore[no-redef, assignment]

from onetl._util.java import try_import_java_class
from onetl._util.sql import clear_statement
from onetl.connection.db_connection.db_connection import DBConnection
from onetl.connection.db_connection.jdbc_connection.dialect import JDBCDialect
@@ -32,6 +33,7 @@
from onetl.connection.db_connection.jdbc_mixin.options import (
JDBCOptions as JDBCMixinOptions,
)
from onetl.exception import MISSING_JVM_CLASS_MSG
from onetl.hooks import slot, support_hooks
from onetl.hwm import Window
from onetl.log import log_lines, log_with_indent
@@ -56,7 +58,7 @@


@support_hooks
class JDBCConnection(JDBCMixin, DBConnection):
class JDBCConnection(JDBCMixin, DBConnection): # noqa: WPS338
user: str
password: SecretStr

@@ -73,6 +75,21 @@ class JDBCConnection(JDBCMixin, DBConnection):
WriteOptions = JDBCWriteOptions
Options = JDBCLegacyOptions

@validator("spark")
def _check_java_class_imported(cls, spark):
try:
try_import_java_class(spark, cls.DRIVER)
except Exception as e:
msg = MISSING_JVM_CLASS_MSG.format(
java_class=cls.DRIVER,
package_source=cls.__name__,
args="",
)
if log.isEnabledFor(logging.DEBUG):
log.debug("Missing Java class", exc_info=e, stack_info=True)
raise ValueError(msg) from e
return spark

@slot
def sql(
self,
22 changes: 3 additions & 19 deletions onetl/connection/db_connection/jdbc_mixin/connection.py
Original file line number Diff line number Diff line change
@@ -12,11 +12,11 @@
from onetl.impl.generic_options import GenericOptions

try:
from pydantic.v1 import Field, PrivateAttr, SecretStr, validator
from pydantic.v1 import Field, PrivateAttr, SecretStr
except (ImportError, AttributeError):
from pydantic import Field, PrivateAttr, SecretStr, validator # type: ignore[no-redef, assignment]
from pydantic import Field, PrivateAttr, SecretStr # type: ignore[no-redef, assignment]

from onetl._util.java import get_java_gateway, try_import_java_class
from onetl._util.java import get_java_gateway
from onetl._util.spark import get_spark_version, stringify
from onetl._util.sql import clear_statement
from onetl._util.version import Version
@@ -27,7 +27,6 @@
from onetl.connection.db_connection.jdbc_mixin.options import (
JDBCOptions as JDBCMixinOptions,
)
from onetl.exception import MISSING_JVM_CLASS_MSG
from onetl.hooks import slot, support_hooks
from onetl.log import log_lines

@@ -285,21 +284,6 @@ def execute(
log.info("|%s| Execution succeeded, nothing returned", self.__class__.__name__)
return df

@validator("spark")
def _check_java_class_imported(cls, spark):
try:
try_import_java_class(spark, cls.DRIVER)
except Exception as e:
msg = MISSING_JVM_CLASS_MSG.format(
java_class=cls.DRIVER,
package_source=cls.__name__,
args="",
)
if log.isEnabledFor(logging.DEBUG):
log.debug("Missing Java class", exc_info=e, stack_info=True)
raise ValueError(msg) from e
return spark

def _query_on_driver(
self,
query: str,