Skip to content

Fixing the native build so it can find the TSL headers #534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 4, 2024
Merged

Fixing the native build so it can find the TSL headers #534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions ...core/tensorflow-core-generator/src/main/java/org/tensorflow/generator/op/OpGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import java.util.Map;
import java.util.Scanner;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.bytedeco.javacpp.BytePointer;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.tensorflow.internal.c_api.TF_ApiDefMap;
Expand Down Expand Up @@ -72,7 +73,7 @@ public final class OpGenerator {

private static final String DEFAULT_OP_DEF_FILE = "org/tensorflow/ops.pbtxt";

private static final Scanner USER_PROMPT = new Scanner(System.in);
private static final Scanner USER_PROMPT = new Scanner(System.in, StandardCharsets.UTF_8);

/**
* Args should be {@code <outputDir> <opDefFile> [base_package]}.
Expand Down Expand Up @@ -195,7 +196,8 @@ private static String arg(String[] args, int idx) {
private static OpList readOpList(String filename, InputStream protoInput) {
try {
if (filename.endsWith(".pbtxt")) {
return TextFormat.parse(new String(protoInput.readAllBytes()), OpList.class);
return TextFormat.parse(
new String(protoInput.readAllBytes(), StandardCharsets.UTF_8), OpList.class);
}
return OpList.parseFrom(protoInput);

Expand Down Expand Up @@ -286,9 +288,8 @@ private void mergeBaseApiDefs(TF_ApiDefMap apiDefMap, TF_Status status) {
}

private void mergeApiDefs(TF_ApiDefMap apiDefMap, TF_Status status) {
try {
Files.walk(apiDefsPath)
.filter(p -> p.toString().endsWith(".pbtxt"))
try (Stream<Path> s = Files.walk(apiDefsPath)) {
s.filter(p -> p.toString().endsWith(".pbtxt"))
.forEach(
p -> {
try {
Expand Down Expand Up @@ -366,7 +367,7 @@ private void createApiDef(OpDef opDef, File apiDefFile) throws IOException {
if (!apiDefFile.exists() && !apiDefFile.createNewFile()) {
System.err.println("Cannot create API definition file \"" + apiDefFile.getPath() + "\"");
}
try (var apiDefWriter = new FileWriter(apiDefFile)) {
try (var apiDefWriter = new FileWriter(apiDefFile, StandardCharsets.UTF_8)) {
var apiDefs = ApiDefs.newBuilder();
apiDefs.addOp(apiDef.build());
apiDefWriter.write(TextFormat.printer().printToString(apiDefs.build()));
Expand Down
12 changes: 12 additions & 0 deletions ...nerator/src/main/java/org/tensorflow/generator/op/javadoc/JavaDocNodeRendererContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
public interface JavaDocNodeRendererContext {

/**
* Encode a URL into a String.
*
* @param url to be encoded
* @return an encoded URL (depending on the configuration)
*/
Expand All @@ -26,11 +28,15 @@ public interface JavaDocNodeRendererContext {
Map<String, String> extendAttributes(Node node, String tagName, Map<String, String> attributes);

/**
* Gets the HTML writer.
*
* @return the HTML writer to use
*/
JavaDocWriter getWriter();

/**
* The HTML for a line break.
*
* @return HTML that should be rendered for a soft line break
*/
String getSoftbreak();
Expand All @@ -45,17 +51,23 @@ public interface JavaDocNodeRendererContext {
void render(Node node);

/**
* Should HTML be escaped?
*
* @return whether HTML blocks and tags should be escaped or not
*/
boolean shouldEscapeHtml();

/**
* Should URLs be sanitized?
*
* @return true if the {@link UrlSanitizer} should be used.
* @since 0.14.0
*/
boolean shouldSanitizeUrls();

/**
* Gets the URL sanitizer.
*
* @return Sanitizer to use for securing {@link Link} href and {@link Image} src if {@link
* #shouldSanitizeUrls()} is true.
* @since 0.14.0
Expand Down
1 change: 1 addition & 0 deletions tensorflow-core/tensorflow-core-native/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@
<includePath>${native.source.directory}/org/tensorflow/internal/c_api/</includePath>
<!-- additional include paths in case of a full native build -->
<includePath>${project.basedir}/bazel-${project.artifactId}/external/org_tensorflow/</includePath>
<includePath>${project.basedir}/bazel-${project.artifactId}/external/local_tsl/</includePath>
<includePath>${project.basedir}/bazel-bin/external/org_tensorflow/</includePath>
<includePath>${project.basedir}/bazel-${project.artifactId}/external/com_google_absl/</includePath>
<includePath>${project.basedir}/bazel-${project.artifactId}/external/eigen_archive/</includePath>
Expand Down
6 changes: 3 additions & 3 deletions .../tensorflow-core-native/src/gen/java/org/tensorflow/internal/c_api/global/tensorflow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
public class tensorflow extends org.tensorflow.internal.c_api.presets.tensorflow {
static { Loader.load(); }

// Parsed from tensorflow/tsl/platform/ctstring_internal.h
// Parsed from tsl/platform/ctstring_internal.h

/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Expand Down Expand Up @@ -165,7 +165,7 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
// #endif // TENSORFLOW_TSL_PLATFORM_CTSTRING_INTERNAL_H_


// Parsed from tensorflow/tsl/platform/ctstring.h
// Parsed from tsl/platform/ctstring.h

/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Expand Down Expand Up @@ -274,7 +274,7 @@ public static native void TF_TString_Copy(TF_TString dst, String src,
// #endif // TENSORFLOW_TSL_PLATFORM_CTSTRING_H_


// Parsed from tensorflow/tsl/c/tsl_status.h
// Parsed from tsl/c/tsl_status.h

/* Copyright 2019 The TensorFlow Authors. All Rights Reserved.

Expand Down
8 changes: 5 additions & 3 deletions ...ensorflow-core-native/src/main/java/org/tensorflow/internal/c_api/presets/tensorflow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,11 @@
value = {"linux", "macosx", "windows"},
compiler = "cpp17",
include = {
"tensorflow/tsl/platform/ctstring_internal.h",
"tensorflow/tsl/platform/ctstring.h",
"tensorflow/tsl/c/tsl_status.h",
// TSL headers are in different places in a bazel build and the downloaded whl
// The lower part is still the same, so multiple roots are set in the pom file.
"tsl/platform/ctstring_internal.h",
"tsl/platform/ctstring.h",
"tsl/c/tsl_status.h",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see a root in the pom file that has the tensorflow/ suffix, to replace the existing paths, is this still required?

"tensorflow/c/c_api_macros.h",
"tensorflow/c/tf_datatype.h",
"tensorflow/c/tf_status.h",
Expand Down
Loading