From 22323504378dd201af14d5dccb0fd8e4011573e2 Mon Sep 17 00:00:00 2001 From: Swift Kim Date: Wed, 2 Nov 2022 17:18:17 +0900 Subject: [PATCH] [Tizen] Add CI related files --- .github/workflows/build-docker.yml | 21 +++ .github/workflows/build.yml | 230 ++++++++++++++++++++++++++++ .github/workflows/check-symbols.yml | 65 ++++++++ ci/tizen/.gitignore | 3 + ci/tizen/Dockerfile | 32 ++++ ci/tizen/arm.patch | 15 ++ ci/tizen/build_llvm.sh | 45 ++++++ ci/tizen/check_symbols.py | 77 ++++++++++ ci/tizen/generate_sysroot.py | 167 ++++++++++++++++++++ 9 files changed, 655 insertions(+) create mode 100644 .github/workflows/build-docker.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/check-symbols.yml create mode 100644 ci/tizen/.gitignore create mode 100644 ci/tizen/Dockerfile create mode 100644 ci/tizen/arm.patch create mode 100755 ci/tizen/build_llvm.sh create mode 100755 ci/tizen/check_symbols.py create mode 100755 ci/tizen/generate_sysroot.py diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000000000..f0e745830dc7f --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,21 @@ +name: Build Docker + +on: [workflow_dispatch] + +jobs: + builder: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/build-push-action@v3 + with: + context: ci/tizen + file: ci/tizen/Dockerfile + push: true + tags: ghcr.io/${{ github.repository_owner }}/build-engine:latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000000..5cc2fded3e1f1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,230 @@ +name: Build + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + container: + image: ghcr.io/flutter-tizen/build-engine:latest + credentials: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + strategy: + matrix: + arch: [arm, arm64, x86] + mode: [debug, release, profile] + include: + - arch: arm + triple: arm-linux-gnueabi + - arch: arm64 + triple: aarch64-linux-gnu + - arch: x86 + triple: i686-linux-gnu + exclude: + - arch: x86 + mode: release + - arch: x86 + mode: profile + + steps: + - uses: actions/checkout@v3 + with: + path: src/flutter + fetch-depth: 0 + + - uses: actions/cache@v3 + with: + path: /tizen_tools/sysroot + key: sysroot + + - name: Install depot_tools + run: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + echo "$PWD/depot_tools" >> $GITHUB_PATH + + - name: Run gclient sync + run: | + gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine + gclient setdep --var=download_dart_sdk=False --var=download_android_deps=False --deps-file=src/flutter/DEPS + gclient sync -v --no-history --shallow + + - name: Generate sysroot + run: src/flutter/ci/tizen/generate_sysroot.py --out /tizen_tools/sysroot + + - name: Build + run: | + src/flutter/tools/gn \ + --target-os linux \ + --linux-cpu ${{ matrix.arch }} \ + --no-goma \ + --target-toolchain /tizen_tools/toolchains \ + --target-sysroot /tizen_tools/sysroot/${{ matrix.arch }} \ + --target-triple ${{ matrix.triple }} \ + --runtime-mode ${{ matrix.mode }} \ + --enable-fontconfig \ + --disable-desktop-embeddings \ + --target-dir build + ninja -C src/out/build flutter_engine_library + cp -f src/third_party/icu/flutter/icudtl.dat src/out/build + + - name: Build gen_snapshot + if: ${{ matrix.mode != 'debug' }} + run: ninja -C src/out/build clang_x64/gen_snapshot + + - uses: actions/upload-artifact@v3 + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }} + path: | + src/out/build/libflutter_engine.so + src/out/build/icudtl.dat + if-no-files-found: error + + - uses: actions/upload-artifact@v3 + if: ${{ github.event_name == 'push' }} + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_symbols + path: src/out/build/so.unstripped/libflutter_engine.so + if-no-files-found: error + + - uses: actions/upload-artifact@v3 + if: ${{ matrix.mode != 'debug' }} + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_linux-x64 + path: src/out/build/clang_x64/gen_snapshot + if-no-files-found: error + + windows-build: + runs-on: windows-latest + + strategy: + matrix: + arch: [arm, arm64] + mode: [release, profile] + + steps: + - name: Run git checkout + run: | + mkdir C:\workspace\engine\src\flutter + cd C:\workspace\engine\src\flutter + git config --global core.autocrlf true + git init --quiet + git remote add origin https://github.com/${{ github.repository }} + git fetch --depth 1 origin ${{ github.sha }} + git checkout FETCH_HEAD + + - name: Environment setup + run: | + Set-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -Force + echo "DEPOT_TOOLS_WIN_TOOLCHAIN=0" >> $Env:GITHUB_ENV + echo "GYP_MSVS_OVERRIDE_PATH=C:\Program Files\Microsoft Visual Studio\2022\Enterprise" >> $Env:GITHUB_ENV + echo "WINDOWSSDKDIR=C:\Program Files (x86)\Windows Kits\10" >> $Env:GITHUB_ENV + + - name: Install depot_tools + run: | + Invoke-WebRequest -Uri https://storage.googleapis.com/chrome-infra/depot_tools.zip -OutFile depot_tools.zip + 7z x -y -o"C:\workspace\depot_tools" .\depot_tools.zip + echo "C:\workspace\depot_tools" >> $Env:GITHUB_PATH + + - name: Run gclient sync + working-directory: C:\workspace\engine + run: | + gclient config --name=src\flutter --unmanaged https://github.com/flutter-tizen/engine + gclient setdep --var=download_dart_sdk=False --deps-file=src/flutter/DEPS + gclient sync -v --no-history --shallow + + - name: Build + working-directory: C:\workspace\engine\src + run: | + python3 .\flutter\tools\gn ` + --linux ` + --linux-cpu=${{ matrix.arch }} ` + --runtime-mode=${{ matrix.mode }} ` + --no-goma ` + --target-dir build + ninja -C .\out\build gen_snapshot + + - uses: actions/upload-artifact@v3 + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_windows-x64 + path: C:\workspace\engine\src\out\build\gen_snapshot.exe + if-no-files-found: error + + macos-build: + runs-on: macos-latest + + strategy: + matrix: + arch: [arm, arm64] + mode: [release, profile] + + steps: + - uses: actions/checkout@v3 + with: + path: src/flutter + fetch-depth: 0 + + - name: Install depot_tools + run: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + echo "$PWD/depot_tools" >> $GITHUB_PATH + + - name: Run gclient sync + run: | + gclient config --name=src/flutter --unmanaged https://github.com/flutter-tizen/engine + gclient setdep --var=download_dart_sdk=False --var=download_android_deps=False --deps-file=src/flutter/DEPS + gclient sync -v --no-history --shallow + + - name: Build + run: | + # Change host_toolchain to mac/clang_x64. + sed -i "" "s|//build/toolchain/linux:clang_$host_cpu|//build/toolchain/mac:clang_$host_cpu|g" src/build/config/BUILDCONFIG.gn + + # Pass dummy values to prevent using the default (Linux) toolchain. + src/flutter/tools/gn \ + --linux \ + --linux-cpu=${{ matrix.arch }} \ + --no-goma \ + --target-toolchain _ \ + --target-sysroot _ \ + --target-triple _ \ + --runtime-mode=${{ matrix.mode }} \ + --disable-desktop-embeddings \ + --target-dir build + ninja -C src/out/build clang_x64/gen_snapshot + + - uses: actions/upload-artifact@v3 + with: + name: tizen-${{ matrix.arch }}-${{ matrix.mode }}_darwin-x64 + path: src/out/build/clang_x64/gen_snapshot + if-no-files-found: error + + release: + needs: [build, windows-build, macos-build] + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/download-artifact@v3 + + - name: Create archives + run: | + for name in tizen-*; do + 7z a $name.zip ./$name/* + done + + - name: Set variables + run: | + echo "SHORT_SHA=$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV + echo "VERSION=$(echo "${{ github.ref_name }}" | cut -d'-' -f2)" >> $GITHUB_ENV + + - uses: softprops/action-gh-release@v1 + with: + name: ${{ env.VERSION }} (${{ env.SHORT_SHA }}) + tag_name: ${{ env.SHORT_SHA }} + target_commitish: ${{ github.sha }} + files: tizen-*.zip + body: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/check-symbols.yml b/.github/workflows/check-symbols.yml new file mode 100644 index 0000000000000..a144bdc12fbef --- /dev/null +++ b/.github/workflows/check-symbols.yml @@ -0,0 +1,65 @@ +name: Check symbols + +on: + workflow_run: + workflows: + - Build + types: + - completed + +jobs: + check: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/checkout@v3 + with: + repository: flutter-tizen/tizen_allowlist + token: ${{ secrets.TIZENAPI_TOKEN }} + path: tizen_allowlist + + - name: Download artifacts + uses: TizenAPI/tizenfx-build-actions/download-workflow-artifacts@master + with: + token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + name: tizen-arm-release + path: artifacts + + - name: Check symbols + run: | + python3 ci/tizen/check_symbols.py \ + --allowlist=tizen_allowlist/4.0.0_native_whitelist_wearable_v12.txt \ + artifacts/libflutter_engine.so + + - name: Commit success status + if: ${{ success() }} + uses: actions/github-script@v6 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.workflow_run.head_sha, + context: 'Check symbols', + state: 'success', + description: 'All symbols are valid', + }); + + - name: Commit failure status + if: ${{ failure() }} + uses: actions/github-script@v6 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.payload.workflow_run.head_sha, + context: 'Check symbols', + state: 'failure', + description: 'Failed in checking symbols', + target_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}', + }); diff --git a/ci/tizen/.gitignore b/ci/tizen/.gitignore new file mode 100644 index 0000000000000..bfa72ad43fb0c --- /dev/null +++ b/ci/tizen/.gitignore @@ -0,0 +1,3 @@ +/llvm-project*/ +/toolchains/ +/sysroot/ diff --git a/ci/tizen/Dockerfile b/ci/tizen/Dockerfile new file mode 100644 index 0000000000000..7f380950d5bed --- /dev/null +++ b/ci/tizen/Dockerfile @@ -0,0 +1,32 @@ +############################### +### Stage for building LLVM ### +############################### + +FROM ubuntu:20.04 AS llvm + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y git zip build-essential cmake ninja-build clang-11 && \ + apt-get clean + +COPY build_llvm.sh . + +RUN /build_llvm.sh + + +############################## +### Create a release image ### +############################## + +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y binutils-arm-linux-gnueabi binutils-aarch64-linux-gnu binutils-i686-linux-gnu && \ + apt-get install -y git curl pkg-config ca-certificates python python3 python3-pip rpm2cpio cpio && \ + apt-get clean + +# Copy build artifacts from the previous stage. +COPY --from=llvm /toolchains/ /tizen_tools/toolchains/ diff --git a/ci/tizen/arm.patch b/ci/tizen/arm.patch new file mode 100644 index 0000000000000..1dabb3f2803d4 --- /dev/null +++ b/ci/tizen/arm.patch @@ -0,0 +1,15 @@ +diff --git a/usr/include/asm-arm/hwcap.h b/usr/include/asm-arm/hwcap.h +index da85060..adaf619 100644 +--- a/usr/include/asm-arm/hwcap.h ++++ b/usr/include/asm-arm/hwcap.h +@@ -26,5 +26,10 @@ + #define HWCAP_VFPD32 (1 << 19) /* set if VFP has 32 regs (not 16) */ + #define HWCAP_IDIV (HWCAP_IDIVA | HWCAP_IDIVT) + ++#define HWCAP2_AES (1 << 0) ++#define HWCAP2_PMULL (1 << 1) ++#define HWCAP2_SHA1 (1 << 2) ++#define HWCAP2_SHA2 (1 << 3) ++#define HWCAP2_CRC32 (1 << 4) + + #endif /* __ASMARM_HWCAP_H */ diff --git a/ci/tizen/build_llvm.sh b/ci/tizen/build_llvm.sh new file mode 100755 index 0000000000000..84b4faf6228fe --- /dev/null +++ b/ci/tizen/build_llvm.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2022 Samsung Electronics Co., Ltd. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +set -e + +SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" +OUTPUT_DIR="$SCRIPT_DIR/toolchains" +cd "$SCRIPT_DIR" + +# Check out the LLVM project source code. +if [ -d llvm-project ]; then + echo "The directory already exists. Skipping download." + cd llvm-project +else + mkdir llvm-project + cd llvm-project + git init + git remote add origin https://github.com/llvm/llvm-project.git + git fetch --depth=1 origin aaaf8e4c409f080f35ea227b20dc6ac8a45c2fa4 + git checkout FETCH_HEAD +fi + +# Run the ninja build. +mkdir -p build && cd build +cmake -G Ninja \ + -DCLANG_VENDOR="Tizen" \ + -DLLVM_ENABLE_PROJECTS="clang" \ + -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64" \ + -DCMAKE_C_COMPILER=clang-11 \ + -DCMAKE_CXX_COMPILER=clang++-11 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$OUTPUT_DIR" \ + ../llvm +ninja install + +# Create symbolic links to binutils. +# See build/toolchain/custom/BUILD.gn for more information. +cd "$OUTPUT_DIR/bin" +for name in ar readelf nm strip; do + ln -sf llvm-$name arm-linux-gnueabi-$name + ln -sf llvm-$name aarch64-linux-gnu-$name + ln -sf llvm-$name i686-linux-gnu-$name +done diff --git a/ci/tizen/check_symbols.py b/ci/tizen/check_symbols.py new file mode 100755 index 0000000000000..e911ee417c5c3 --- /dev/null +++ b/ci/tizen/check_symbols.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright 2022 Samsung Electronics Co., Ltd. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import subprocess +import sys + + +class Symbol: + + def __init__(self, addr, type, name): + self.addr = addr + self.type = type + self.name = name + + def __str__(self): + return '{} {} {}'.format(self.addr, self.type, self.name) + + @staticmethod + def parse(line): + # Format: " U abort@GLIBC_2.4" (addr can be empty) + return Symbol(line[:8].strip(), line[9], line[11:].strip().split('@')[0]) + + +def check_symbol(sofile, allowlist): + if not os.access(sofile, os.R_OK): + sys.exit('{} is not a valid file.'.format(sofile)) + if not os.access(allowlist, os.R_OK): + sys.exit('{} is not a valid file.'.format(allowlist)) + + try: + symbols_raw = subprocess.check_output(['nm', '-gDC', + sofile]).decode('utf-8').splitlines() + symbols = [Symbol.parse(line) for line in symbols_raw] + except subprocess.CalledProcessError as error: + sys.exit('nm failed: {}'.format(error)) + + with open(allowlist, 'r') as file: + allowlist = [line.strip() for line in file.readlines()] + + not_allowed = [] + for symbol in symbols: + if symbol.addr: + continue + if symbol.name.startswith('FlutterEngine'): + continue + if symbol.name in allowlist: + continue + not_allowed.append(symbol) + + if not_allowed: + print('Symbols not allowed ({}):'.format(sofile)) + for symbol in not_allowed: + print(symbol) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--allowlist', type=str, required=True, help='Path to the allowlist file' + ) + parser.add_argument( + 'sofile', type=str, nargs='+', help='Path to the .so file' + ) + args = parser.parse_args() + + for sofile in args.sofile: + check_symbol(sofile, args.allowlist) + + +# Execute only if run as a script. +if __name__ == '__main__': + main() diff --git a/ci/tizen/generate_sysroot.py b/ci/tizen/generate_sysroot.py new file mode 100755 index 0000000000000..9a99948f276bd --- /dev/null +++ b/ci/tizen/generate_sysroot.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +# Copyright 2022 Samsung Electronics Co., Ltd. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import argparse +import os +import re +import shutil +import subprocess +import sys +import urllib.parse +import urllib.request +from pathlib import Path + +base_packages = [ + 'gcc', + 'glibc', + 'glibc-devel', + 'libgcc', + 'linux-glibc-devel', + 'zlib-devel', +] + +unified_packages = [ + 'freetype2-devel', + 'libpng-devel', +] + + +def generate_sysroot(sysroot: Path, api_version: float, arch: str, quiet=False): + if arch == 'arm': + tizen_arch = 'armv7l' + elif arch == 'arm64': + tizen_arch = 'aarch64' + elif arch == 'x86': + tizen_arch = 'i686' + else: + sys.exit('Unknown arch: ' + arch) + + base_repo = 'http://download.tizen.org/snapshots/tizen/{}-base/latest/repos/standard/packages'.format( + api_version + ) + unified_repo = 'http://download.tizen.org/snapshots/tizen/{}-unified/latest/repos/standard/packages'.format( + api_version + ) + + # Retrieve html documents. + documents = {} + for url in ['{}/{}'.format(base_repo, tizen_arch), + '{}/{}'.format(base_repo, 'noarch'), '{}/{}'.format(unified_repo, + tizen_arch), + '{}/{}'.format(unified_repo, 'noarch')]: + request = urllib.request.Request(url) + with urllib.request.urlopen(request) as response: + documents[url] = response.read().decode('utf-8') + + # Download packages. + download_path = sysroot / '.rpms' + download_path.mkdir(exist_ok=True) + existing_rpms = [f for f in download_path.iterdir() if f.suffix == '.rpm'] + + for package in base_packages + unified_packages: + quoted = urllib.parse.quote(package) + pattern = re.escape(quoted) + '-\\d+\\.[\\d_\\.]+-[\\d\\.]+\\..+\\.rpm' + + if any([re.match(pattern, rpm.name) for rpm in existing_rpms]): + continue + + for parent, doc in documents.items(): + match = re.search('.+?'.format(pattern), doc) + if match: + rpm_url = '{}/{}'.format(parent, match.group(1)) + break + + if match: + if not quiet: + print('Downloading ' + rpm_url) + urllib.request.urlretrieve(rpm_url, download_path / match.group(1)) + else: + sys.exit('Could not find a package named ' + package) + + # Extract files. + for rpm in [f for f in download_path.iterdir() if f.suffix == '.rpm']: + command = 'rpm2cpio {} | cpio -idum --quiet'.format(rpm) + subprocess.run(command, shell=True, cwd=sysroot, check=True) + + # Create symbolic links. + asm = sysroot / 'usr' / 'include' / 'asm' + if not asm.exists(): + os.symlink('asm-' + arch, asm) + pkgconfig = sysroot / 'usr' / 'lib' / 'pkgconfig' + if arch == 'arm64' and not pkgconfig.exists(): + os.symlink('../lib64/pkgconfig', pkgconfig) + + # Copy objects required by the linker, such as crtbeginS.o and libgcc.a. + if arch == 'arm64': + libpath = sysroot / 'usr' / 'lib64' + else: + libpath = sysroot / 'usr' / 'lib' + subprocess.run( + 'cp gcc/*/*/*.o gcc/*/*/*.a .', shell=True, cwd=libpath, check=True + ) + + # Apply a patch if applicable. + patch = Path(__file__).parent / '{}.patch'.format(arch) + if patch.is_file(): + command = 'patch -p1 -s -d {} < {}'.format(sysroot, patch) + subprocess.run(command, shell=True, check=True) + + +def main(): + # Check dependencies. + for dep in ['rpm2cpio', 'cpio', 'git']: + if not shutil.which(dep): + sys.exit( + '{} is not installed. To install, run:\n' + ' sudo apt install {}'.format(dep, dep) + ) + + # Parse arguments. + parser = argparse.ArgumentParser(description='Tizen sysroot generator') + parser.add_argument( + '-o', + '--out', + metavar='PATH', + type=str, + help='Path to the output directory' + ) + parser.add_argument( + '-f', + '--force', + action='store_true', + help='Force re-downloading of packages' + ) + parser.add_argument( + '-q', '--quiet', action='store_true', help='Suppress log output' + ) + parser.add_argument( + '--api-version', + metavar='VER', + default=5.5, + type=float, + help='Target API version (defaults to 5.5)' + ) + args = parser.parse_args() + + if args.out: + outpath = Path(args.out) + else: + outpath = Path(__file__).parent / 'sysroot' + outpath.mkdir(exist_ok=True) + + for arch in ['arm', 'arm64', 'x86']: + sysroot = outpath / arch + if args.force and sysroot.is_dir(): + shutil.rmtree(sysroot) + sysroot.mkdir(exist_ok=True) + + if not args.quiet: + print('Generating sysroot for {}...'.format(arch)) + generate_sysroot(sysroot.resolve(), args.api_version, arch, args.quiet) + + +# Execute only if run as a script. +if __name__ == '__main__': + main()