Skip to content

Commit

Permalink
Merge pull request #24827 Add gradle files for typescript containers …
Browse files Browse the repository at this point in the history
…and update the version info.
  • Loading branch information
robertwb authored Dec 30, 2022
2 parents 0bba439 + 3bd5f78 commit 889fd8b
Show file tree
Hide file tree
Showing 12 changed files with 177 additions and 68 deletions.
3 changes: 2 additions & 1 deletion build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ tasks.rat {

// Tour Of Beam backend autogenerated Datastore indexes
"learning/tour-of-beam/backend/internal/storage/index.yaml",

// Tour Of Beam backend autogenerated Playground GRPC API stubs and mocks
"learning/tour-of-beam/backend/playground_api/api.pb.go",
"learning/tour-of-beam/backend/playground_api/api_grpc.pb.go",
Expand Down Expand Up @@ -553,6 +553,7 @@ tasks.register("pushAllDockerImages") {
dependsOn(":sdks:java:container:pushAll")
dependsOn(":sdks:python:container:pushAll")
dependsOn(":sdks:go:container:pushAll")
dependsOn(":sdks:typescript:container:pushAll")
for (version in project.ext.get("allFlinkVersions") as Array<*>) {
dependsOn(":runners:flink:${version}:job-server-container:dockerPush")
}
Expand Down
2 changes: 2 additions & 0 deletions release/src/main/scripts/set_version.sh
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ if [[ -z "$IS_SNAPSHOT_VERSION" ]] ; then
sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}'/" sdks/python/apache_beam/version.py
sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION/" gradle.properties
sed -i -e "s/SdkVersion = .*/SdkVersion = \"$TARGET_VERSION\"/" sdks/go/pkg/beam/core/core.go
sed -i -e "s/\"version\": .*/\"version\": \"$TARGET_VERSION\",/" sdks/typescript/package.json
else
# For snapshot version:
# Java/gradle appends -SNAPSHOT
Expand All @@ -87,6 +88,7 @@ else
sed -i -e "s/^__version__ = .*/__version__ = '${TARGET_VERSION}.dev'/" sdks/python/apache_beam/version.py
sed -i -e "s/sdk_version=.*/sdk_version=$TARGET_VERSION.dev/" gradle.properties
sed -i -e "s/SdkVersion = .*/SdkVersion = \"${TARGET_VERSION}.dev\"/" sdks/go/pkg/beam/core/core.go
sed -i -e "s/\"version\": .*/\"version\": \"$TARGET_VERSION-SNAPSHOT\",/" sdks/typescript/package.json
fi

if [[ "$GIT_ADD" == yes ]] ; then
Expand Down
67 changes: 67 additions & 0 deletions sdks/typescript/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

plugins { id 'org.apache.beam.module' }

description = "Apache Beam :: SDKs :: Typescript"

configurations {
packTarball
}

def tarballName = "apache-beam.tgz"

def typescriptSdkDeps = project.files(
project.fileTree(
dir: "${project.rootDir}",
include: ['model/**', 'sdks/typescript/src/**', 'sdks/typescript/*.json'],
// Exclude temporary directories and files that are generated
// during build and test.
exclude: [
'**/build/**',
'**/dist/**',
'**/target/**',
'**/.gogradle/**',
'**/reports/test/index.html',
])
)

def pack = tasks.register("pack") {

doLast {
// Build artifact
exec {
executable 'sh'
args '-c', "npm install && npm run build && npm pack --pack-destination ${buildDir}"
}

def collection = fileTree(buildDir){ include "**/*${project.version}*.tgz" exclude 'srcs/**'}

// we need a fixed name for the artifact
copy { from collection.singleFile; into buildDir; rename { tarballName } }
logger.info('Create distribution tar file {} in {}', tarballName, buildDir)
}
inputs.files(typescriptSdkDeps)
.withPropertyName('typescriptSdkDeps')
.withPathSensitivity(PathSensitivity.RELATIVE)
outputs.file "${buildDir}/${tarballName}"
}

artifacts {
packTarball file: file("${buildDir}/${tarballName}"), builtBy: pack
}
5 changes: 5 additions & 0 deletions sdks/typescript/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@

set -e

# Make the packaging version available to the code.
echo "export const version = \"$npm_package_version\";" > src/apache_beam/version.ts

# Using npx to execute ttsc from the local node_modules environment.
npx ttsc -p .

# Copy the python bootstrap script.
mkdir -p dist/resources
cp ../java/extensions/python/src/main/resources/org/apache/beam/sdk/extensions/python/bootstrap_beam_venv.py dist/resources
24 changes: 0 additions & 24 deletions sdks/typescript/build_container.sh

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,10 @@ ENV NODE_ENV=development
WORKDIR /app

# Copy beam package.
COPY apache-beam-0.38.0.tgz ./apache-beam-0.38.0.tgz
COPY target/apache-beam.tgz ./apache-beam.tgz

# Install dependencies and compile
RUN npm install apache-beam-0.38.0.tgz
RUN npm install apache-beam.tgz

# Check that filesystem is set up as expected
RUN ls -a

COPY boot /opt/apache/beam/
COPY target/launcher/linux_amd64/boot /opt/apache/beam/
ENTRYPOINT ["/opt/apache/beam/boot"]
11 changes: 6 additions & 5 deletions sdks/typescript/boot.go → sdks/typescript/container/boot.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"

"github.com/apache/beam/sdks/v2/go/pkg/beam/artifact"
"github.com/apache/beam/sdks/v2/go/pkg/beam/provision"
Expand Down Expand Up @@ -120,7 +121,7 @@ func main() {
}
defer f.Close()
f.WriteString("{\n")
f.WriteString(" \"name\": \"beam-worker\",\n \"version\": \"1.0\",\n")
f.WriteString(" \"name\": \"beam-worker\",\n \"version\": \"1.0.0\",\n")
f.WriteString(" \"overrides\": {\n")
needsComma := false
for pkg, path := range npmOverrides {
Expand Down Expand Up @@ -158,7 +159,6 @@ func main() {

args := []string{
entrypoint,
"--id=" + *id,
"--logging_endpoint=" + *loggingEndpoint,
"--control_endpoint=" + *controlEndpoint,
"--semi_persist_dir=" + *semiPersistDir,
Expand All @@ -169,13 +169,14 @@ func main() {
args = append(args, "--status_endpoint="+info.GetStatusEndpoint().GetUrl())
}

workerIds := append([]string{*workerId}, info.GetSiblingWorkerIds()...)
workerIds := append([]string{*id}, info.GetSiblingWorkerIds()...)
var wg sync.WaitGroup
wg.Add(len(workerIds))
for _, workerId := range workerIds {
go func(workerId string) {
log.Printf("Executing: python %v", strings.Join(args, " "))
log.Fatalf("User program exited: %v", execx.ExecuteEnv(map[string]string{"WORKER_ID": workerId}, "npx", args...))
workerArgs := append(append([]string{}, args...), "--id="+workerId)
log.Printf("Executing: npx %v", strings.Join(workerArgs, " "))
log.Fatalf("User program exited: %v", execx.Execute("npx", workerArgs...))
}(workerId)
}
wg.Wait()
Expand Down
79 changes: 79 additions & 0 deletions sdks/typescript/container/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* License); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an AS IS BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

plugins { id 'org.apache.beam.module' }
applyDockerNature()
applyGoNature()

description = "Apache Beam :: SDKs :: Typescript :: Container"

configurations {
packTarball
}

dependencies {
packTarball project(path: ":sdks:typescript", configuration: "packTarball")
}

goBuild {
goTargets = '*.go' // only build the immediate directory.
outputLocation = './build/target/launcher/${GOOS}_${GOARCH}/boot'
}

def copyDockerfileDependencies = tasks.register("copyDockerfileDependencies", Copy) {
from configurations.packTarball
into "build/target"
if(configurations.packTarball.isEmpty()) {
throw new StopExecutionException("Empty sdk tarball.");
}
}

docker {
name containerImageName(
name: project.docker_image_default_repo_prefix + "typescript_sdk",
root: project.rootProject.hasProperty(["docker-repository-root"]) ?
project.rootProject["docker-repository-root"] :
project.docker_image_default_repo_root,
tag: project.rootProject.hasProperty(["docker-tag"]) ?
project.rootProject["docker-tag"] : project.sdk_version)
// tags used by dockerTag task
tags containerImageTags()
files "./build"
}

dockerPrepare.dependsOn goBuild
dockerPrepare.dependsOn copyDockerfileDependencies

if (project.rootProject.hasProperty(["docker-pull-licenses"])) {
def copyGolangLicenses = tasks.register("copyGolangLicenses", Copy) {
from "${project(':release:go-licenses:py').buildDir}/output"
into "build/target/go-licenses"
dependsOn ':release:go-licenses:py:createLicenses'
}
dockerPrepare.dependsOn copyGolangLicenses
} else {
def skipPullLicenses = tasks.register("skipPullLicenses", Exec) {
executable "sh"
args "-c", "mkdir -p build/target/go-licenses"
}
dockerPrepare.dependsOn skipPullLicenses
}

task pushAll {
dependsOn ":sdks:typescript:container:dockerPush"
}
11 changes: 7 additions & 4 deletions sdks/typescript/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions sdks/typescript/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "apache-beam",
"version": "0.38.0",
"version": "2.45.0-SNAPSHOT",
"devDependencies": {
"@google-cloud/bigquery": "^5.12.0",
"@types/mocha": "^9.0.0",
Expand Down Expand Up @@ -53,6 +53,8 @@
"main": "./dist/src/apache_beam/index.js",
"exports": {
".": "./dist/src/apache_beam/index.js",
"./": "./dist/src/apache_beam/"
"./transforms": "./dist/src/apache_beam/transforms/index.js",
"./runners": "./dist/src/apache_beam/index.js",
"./*": "./dist/src/apache_beam/*.js"
}
}
26 changes: 0 additions & 26 deletions sdks/typescript/src/apache_beam/version.ts

This file was deleted.

2 changes: 2 additions & 0 deletions settings.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,8 @@ include(":sdks:python:test-suites:tox:py37")
include(":sdks:python:test-suites:tox:py38")
include(":sdks:python:test-suites:tox:py39")
include(":sdks:python:test-suites:tox:py310")
include(":sdks:typescript")
include(":sdks:typescript:container")
include(":vendor:bytebuddy-1_12_8")
include(":vendor:grpc-1_48_1")
include(":vendor:calcite-1_28_0")
Expand Down

0 comments on commit 889fd8b

Please sign in to comment.