Skip to content

Commit

Permalink
Add api_key to spark docs, allow api_key to be empty (#908)
Browse files Browse the repository at this point in the history
Signed-off-by: henneberger <git@danielhenneberger.com>
  • Loading branch information
henneberger authored Jan 25, 2021
1 parent dba5440 commit 07d671f
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 6 deletions.
17 changes: 12 additions & 5 deletions experimental/integrations/marquez-spark-agent/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@ from uuid import uuid4

jar = 'marquez-spark-LATEST.jar'
files = [f"gs://bq-airflow-spark/{jar}"]
marquez_path = 'https://marquez.example.org:5000'
marquez_path = 'http://localhost:5000'
run_id = uuid4()
job_name = 'submit_job'
api_key = ''
properties = {
'spark.driver.extraJavaOptions':
f"-javaagent:{jar}={marquez_path}/api/v1/namespaces/foo/job/{job_name}/runs/{run_id}"
f"-javaagent:{jar}={marquez_path}/api/v1/namespaces/foo/job/{job_name}/runs/{run_id}?api_key={api_key}"
}

t1 = DataProcPySparkOperator(
Expand All @@ -47,9 +48,10 @@ marquez_path = 'http://localhost:5000'
run_id = uuid4()
jar_path = 'marquez-spark-LATEST.jar'
job_name = 'submit_job'
api_key = ''
properties = {
'spark.driver.extraJavaOptions':
f"-javaagent:{jar_path}={marquez_path}/api/v1/namespaces/foo/job/{job_name}/runs/{run_id}"
f"-javaagent:{jar_path}={marquez_path}/api/v1/namespaces/foo/job/{job_name}/runs/{run_id}?api_key={api_key}"
}

t1 = SparkSubmitOperator(
Expand All @@ -65,10 +67,10 @@ t1 = SparkSubmitOperator(

## Arguments
The java agent accepts an argument in the form of a uri. It includes the location of Marquez, the
namespace name, the job name, and a unique run id. This run id will be emitted as a parent run
namespace name, the job name, and a unique run id. The run id will be emitted as a parent run
facet.
```
{marquez_home}/api/v1/namespaces/{namespace}/job/{job_name}/runs/{run_uuid}"
{marquez_home}/api/v1/namespaces/{namespace}/job/{job_name}/runs/{run_uuid}?api_key={api_key}"
```
For example:
Expand All @@ -78,6 +80,11 @@ https://marquez.example.com:5000/api/v1/namespaces/foo/job/spark.submit_job/runs

# Build

## Java 8
Testing requires a Java 8 JVM to test the scala spark components.

`export JAVA_HOME=`/usr/libexec/java_home -v 1.8`

## Testing
To run the tests, run:
```sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static ArgumentParser parse(String agentArgs) {
private static String getApiKey(List<NameValuePair> nameValuePairList) {
String apiKey;
if ((apiKey = getNamedParameter(nameValuePairList, "api_key")) != null) {
return apiKey;
return apiKey.isEmpty() ? null : apiKey;
}
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ public static Collection<Object[]> data() {
"ea445b5c-22eb-457a-8007-01c7c52b6e54",
Optional.empty()
});
pass.add(
new Object[] {
"http://localhost:5000/api/v1/namespaces/ns_name/jobs/job_name/runs/ea445b5c-22eb-457a-8007-01c7c52b6e54?api_key=",
"http://localhost:5000",
"v1",
"ns_name",
"job_name",
"ea445b5c-22eb-457a-8007-01c7c52b6e54",
Optional.empty()
});
return pass;
}

Expand Down

0 comments on commit 07d671f

Please sign in to comment.