[HOPSWORKS-3326][Append] Timezone-independent conversion of Hudi Time…

…stamps and Timezone-Sensitive GH-Actions (#828)

* made conversion from utc explicit

* windows and timezone action

* rename action

* change runner to windows

* removed python variants from tz-local actions

* added java tz local action

* small rename

* added tests

* remove unused variable

* set correct timezone in java tests

.github/workflows/java-ut.yml

@@ -3,11 +3,43 @@ name: java
 on: pull_request
 
 jobs:
-  unit_tests:
+  unit_tests_utc:
     name: Java Unit Tests
     runs-on: ubuntu-latest
 
     steps:
+      - name: Set Timezone
+        run: sudo timedatectl set-timezone UTC
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: Test
+        working-directory: ./java
+        run: mvn --batch-mode test
+
+  unit_tests_local:
+    name: Java Unit Tests (Local TZ)
+    runs-on: ubuntu-latest
+
+    steps:      
+      - name: Set Timezone
+        run: sudo timedatectl set-timezone Europe/Amsterdam
+
       - name: Checkout
         uses: actions/checkout@v3
 

.github/workflows/python-lint.yml

@@ -27,15 +27,18 @@ jobs:
       - name: end-of-file-fixer
         run: end-of-file-fixer $(find python -name "*.py" -type f) || exit 1
 
-  unit_tests:
-    name: Unit Testing
+  unit_tests_ubuntu_utc:
+    name: Unit Testing (Ubuntu)
     needs: lint_stylecheck
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.7", "3.8", "3.9"]
 
-    steps:
+    steps:      
+      - name: Set Timezone
+        run: sudo timedatectl set-timezone UTC
+
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         name: Setup Python
@@ -50,3 +53,76 @@ jobs:
 
       - name: Run Pytest suite
         run: pytest python/tests
+
+  unit_tests_ubuntu_local:
+    name: Unit Testing (Ubuntu) (Local TZ)
+    needs: lint_stylecheck
+    runs-on: ubuntu-latest
+
+    steps:      
+      - name: Set Timezone
+        run: sudo timedatectl set-timezone Europe/Amsterdam
+
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        name: Setup Python
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "python/setup.py"
+      - run: pip install -e python[python,dev,docs]
+
+      - name: Display Python version
+        run: python --version
+
+      - name: Run Pytest suite
+        run: pytest python/tests
+
+  unit_tests_windows_utc:
+    name: Unit Testing (Windows)
+    needs: lint_stylecheck
+    runs-on: windows-latest
+
+    steps:
+      - name: Set Timezone
+        run: tzutil /s "UTC"
+
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        name: Setup Python
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "python/setup.py"
+      - run: pip install -e python[python,dev,docs]
+
+      - name: Display Python version
+        run: python --version
+
+      - name: Run Pytest suite
+        run: pytest python/tests
+
+
+  unit_tests_windows_local:
+    name: Unit Testing (Windows) (Local TZ)
+    needs: lint_stylecheck
+    runs-on: windows-latest
+
+    steps:
+      - name: Set Timezone
+        run: tzutil /s "W. Europe Standard Time"
+
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        name: Setup Python
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "python/setup.py"
+      - run: pip install -e python[python,dev,docs]
+
+      - name: Display Python version
+        run: python --version
+
+      - name: Run Pytest suite
+        run: pytest python/tests

python/hsfs/util.py

@@ -137,7 +137,7 @@ def get_timestamp_from_date_string(input_date):
 
 
 def get_hudi_datestr_from_timestamp(timestamp):
-    return datetime.fromtimestamp(timestamp / 1000).strftime("%Y%m%d%H%M%S%f")[:-3]
+    return datetime.utcfromtimestamp(timestamp / 1000).strftime("%Y%m%d%H%M%S%f")[:-3]
 
 
 def convert_event_time_to_timestamp(event_time):

python/tests/test_util.py

@@ -0,0 +1,24 @@
+#
+#   Copyright 2022 Hopsworks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+from hsfs import util
+
+
+class TestUtil:
+    def test_get_hudi_datestr_from_timestamp(self):
+        dt = util.get_hudi_datestr_from_timestamp(1640995200000)
+
+        assert dt == "20220101000000000"