From 477f5b77563280a2ffee2252250c80e8fae8fb1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Till=20D=C3=B6hmen?= Date: Wed, 19 Oct 2022 14:19:44 +0200 Subject: [PATCH] [HOPSWORKS-3326][Append] Timezone-independent conversion of Hudi Timestamps and Timezone-Sensitive GH-Actions (#828) * made conversion from utc explicit * windows and timezone action * rename action * change runner to windows * removed python variants from tz-local actions * added java tz local action * small rename * added tests * remove unused variable * set correct timezone in java tests --- .github/workflows/java-ut.yml | 34 ++++++++++++- .github/workflows/python-lint.yml | 82 +++++++++++++++++++++++++++++-- python/hsfs/util.py | 2 +- python/tests/test_util.py | 24 +++++++++ 4 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 python/tests/test_util.py diff --git a/.github/workflows/java-ut.yml b/.github/workflows/java-ut.yml index 21dab9d972..44f8d33ebe 100644 --- a/.github/workflows/java-ut.yml +++ b/.github/workflows/java-ut.yml @@ -3,11 +3,43 @@ name: java on: pull_request jobs: - unit_tests: + unit_tests_utc: name: Java Unit Tests runs-on: ubuntu-latest steps: + - name: Set Timezone + run: sudo timedatectl set-timezone UTC + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'adopt' + + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Test + working-directory: ./java + run: mvn --batch-mode test + + unit_tests_local: + name: Java Unit Tests (Local TZ) + runs-on: ubuntu-latest + + steps: + - name: Set Timezone + run: sudo timedatectl set-timezone Europe/Amsterdam + - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 7d46cb79aa..8f44c4f869 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -27,15 +27,18 @@ jobs: - name: end-of-file-fixer run: end-of-file-fixer $(find python -name "*.py" -type f) || exit 1 - unit_tests: - name: Unit Testing + unit_tests_ubuntu_utc: + name: Unit Testing (Ubuntu) needs: lint_stylecheck runs-on: ubuntu-latest strategy: matrix: python-version: ["3.7", "3.8", "3.9"] - steps: + steps: + - name: Set Timezone + run: sudo timedatectl set-timezone UTC + - uses: actions/checkout@v3 - uses: actions/setup-python@v4 name: Setup Python @@ -50,3 +53,76 @@ jobs: - name: Run Pytest suite run: pytest python/tests + + unit_tests_ubuntu_local: + name: Unit Testing (Ubuntu) (Local TZ) + needs: lint_stylecheck + runs-on: ubuntu-latest + + steps: + - name: Set Timezone + run: sudo timedatectl set-timezone Europe/Amsterdam + + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + name: Setup Python + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "python/setup.py" + - run: pip install -e python[python,dev,docs] + + - name: Display Python version + run: python --version + + - name: Run Pytest suite + run: pytest python/tests + + unit_tests_windows_utc: + name: Unit Testing (Windows) + needs: lint_stylecheck + runs-on: windows-latest + + steps: + - name: Set Timezone + run: tzutil /s "UTC" + + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + name: Setup Python + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "python/setup.py" + - run: pip install -e python[python,dev,docs] + + - name: Display Python version + run: python --version + + - name: Run Pytest suite + run: pytest python/tests + + + unit_tests_windows_local: + name: Unit Testing (Windows) (Local TZ) + needs: lint_stylecheck + runs-on: windows-latest + + steps: + - name: Set Timezone + run: tzutil /s "W. Europe Standard Time" + + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + name: Setup Python + with: + python-version: "3.9" + cache: "pip" + cache-dependency-path: "python/setup.py" + - run: pip install -e python[python,dev,docs] + + - name: Display Python version + run: python --version + + - name: Run Pytest suite + run: pytest python/tests diff --git a/python/hsfs/util.py b/python/hsfs/util.py index f34d96d403..949c933296 100644 --- a/python/hsfs/util.py +++ b/python/hsfs/util.py @@ -137,7 +137,7 @@ def get_timestamp_from_date_string(input_date): def get_hudi_datestr_from_timestamp(timestamp): - return datetime.fromtimestamp(timestamp / 1000).strftime("%Y%m%d%H%M%S%f")[:-3] + return datetime.utcfromtimestamp(timestamp / 1000).strftime("%Y%m%d%H%M%S%f")[:-3] def convert_event_time_to_timestamp(event_time): diff --git a/python/tests/test_util.py b/python/tests/test_util.py new file mode 100644 index 0000000000..dc2df748f1 --- /dev/null +++ b/python/tests/test_util.py @@ -0,0 +1,24 @@ +# +# Copyright 2022 Hopsworks AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from hsfs import util + + +class TestUtil: + def test_get_hudi_datestr_from_timestamp(self): + dt = util.get_hudi_datestr_from_timestamp(1640995200000) + + assert dt == "20220101000000000"