Skip to content

Commit

Permalink
feat: process lib directory in .gama files for ADS projects deploym…
Browse files Browse the repository at this point in the history 
…ent (#3644)

* feat: add processing of lib directory in gama files #3640

* chore: fix sonar bugs #3640
  • Loading branch information
@jgomer2001
jgomer2001 authored Jan 19, 2023
1 parent e572552 commit 40268ad
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 33 deletions.
6 changes: 3 additions & 3 deletions docs/admin/developer/agama/lifecycle.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ You may like to make modifications and enhancements to your flow. There are two
https://<your-host>/jans-config-api/api/v1/agama/source/com.acme.myflow
```

- Applies a series of modifications to the flow `com.acme.myflow`: nullifies its description, sets the value of configuration properties, and modifies the creation timestamp to *Aug 8th 23:06:40 UTC*
- Applies a series of modifications to the flow `com.acme.myflow`: nullifies its description, sets the value of configuration properties, and modifies the creation timestamp to *Aug 8th 2022 23:06:40 UTC*

```
curl -k -i -H 'Authorization: Bearer <token>' -H 'Content-Type: application/json-patch+json'
Expand Down Expand Up @@ -221,7 +221,7 @@ There are two endpoints for retrieval:
**Notes:**

- Ensure the tokens used have scope `https://jans.io/oauth/config/agama.readonly`
- The response of a successful operation returns a 200 status code with a JSON representation of the flow(s). If some fields result unfamiliar to you, consult the swagger (open api) document linked [above](#creating-a-flow)
- The response of a successful operation returns a 200 status code with a JSON representation of the flow(s). If some fields result unfamiliar to you, consult the swagger (open api) document linked [above](#creating-a-flow)
- By default the source code is not included (this may clutter the output considerably). Append `?includeSource=true` to the endpoint URL to have the source in the output

Example:
Expand Down Expand Up @@ -262,4 +262,4 @@ By design Agama is a [transpiled language](./dsl.md#language-compiler) and trans
- If this was the first transpilation attempt, i.e. it's a recently created flow, a message like "Source code has errors" will appear in the browser when launching the flow
- Otherwise, no error is shown and the flow will behave as if no changes had been applied to the flow's code. This helps preserve the last known "healthy" state of your flow so end-users are not impacted

In any case, the cause of the error can be inspected by [retrieving](#flow-retrieval-and-removal) the flow's data and checking the property `codeError`.
In any case, the cause of the error can be inspected by [retrieving](#flow-retrieval-and-removal) the flow's data and checking the property `codeError`.
155 changes: 125 additions & 30 deletions jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import org.slf4j.Logger;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;

/*
* This bean deploys .gama project files. Modifications of this file must not only account a single
Expand All @@ -61,11 +62,18 @@
public class Deployer {

private static final String BASE_DN = "ou=deployments,ou=agama,o=jans";
private static final String CUST_LIBS_DIR = "/opt/jans/jetty/jans-auth/custom/libs";
private static final String ASSETS_DIR = "/opt/jans/jetty/jans-auth/agama";

private static final String[] ASSETS_SUBDIRS = { "ftl", "fl" };
private static final String SCRIPTS_SUBDIR = "scripts";

private static final String[] TEMPLATES_EXTENSIONS = new String[] { "ftl", "ftlh" };
private static final String[] SCRIPTS_EXTENSIONS = new String[] { "java", "groovy" };
private static final String FLOW_EXT = "flow";

private static final String METADATA_FILE = "project.json";
private static final boolean ON_CONTAINERS = System.getenv("CN_VERSION") != null;

@Inject
private ObjectMapper mapper;
Expand All @@ -88,6 +96,7 @@ public class Deployer {
private Map<String, Long> projectsFinishTimes;
private Map<String, Set<String>> projectsBasePaths;
private Map<String, Set<String>> projectsFlows;
private Map<String, Set<String>> projectsLibs;

public void process() throws IOException {

Expand Down Expand Up @@ -125,14 +134,15 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept
logger.info("Marking deployment task as active");
//This merge helps other nodes/pods not to take charge of this very deployment task
entryManager.merge(dep);

//Check the zip has the expected layout

Path p = extractGamaFile(b64EncodedAssets);
String tmpdir = p.toString();
dd.setProjectMetadata(computeMetadata(name, tmpdir));

//Check the zip has the expected layout
Path pcode = Paths.get(tmpdir, "code");
Path pweb = Paths.get(tmpdir, "web");
Path plib = Paths.get(tmpdir, "lib");

if (Files.isDirectory(pcode) && Files.isDirectory(pweb)) {

Expand All @@ -141,17 +151,24 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept
if (dd.getError() == null) {
projectsFlows.put(prjId, flowIds);

ZipFile zip = compileAssetsArchive(p, pweb);
byte[] bytes = extractZipFileWithPurge(zip, ASSETS_DIR, projectsBasePaths.get(prjId));
Set<String> libsPaths = transferJarFiles(plib);
ZipFile zip = compileAssetsArchive(p, pweb, plib);
byte[] bytes = extractZipFileWithPurge(zip, ASSETS_DIR,
projectsBasePaths.get(prjId), projectsLibs.get(prjId));

Set<String> basePaths = new HashSet<>();
//Update this project's base paths: use the subdirs of web folder
Files.find​(pweb, 1, (pa, attrs) -> attrs.isDirectory())
.map(pa -> pa.getFileName().toString()).forEach(basePaths::add);
basePaths.remove(pweb.getFileName().toString());
projectsBasePaths.put(prjId, basePaths);


//Update this project's libs paths
libsPaths.addAll(computeSourcePaths(plib));
projectsLibs.put(prjId, libsPaths);

dd.setFolders(new ArrayList<>(basePaths));
dd.setLibs(new ArrayList<>(libsPaths));
//Update binary in DB - not a gama file anymore!
dep.setAssets(new String(b64Encoder.encode(bytes), UTF_8));
}
Expand Down Expand Up @@ -195,12 +212,14 @@ private Set<String> createFlows(Path dir, DeploymentDetails dd) throws IOExcepti
(path, attrs) -> attrs.isRegularFile() && path.getFileName().toString().endsWith("." + FLOW_EXT);

logger.info("Looking for .{} files under {}", FLOW_EXT, dir);
Map<Path, String> flowsCode = Files.find​(dir, 2, matcher).collect(Collectors.toMap(p -> p, p -> ""));
Map<Path, String> flowsCode = Files.find​(dir, 3, matcher).collect(Collectors.toMap(p -> p, p -> ""));
flowsCode = new HashMap<>(flowsCode); //Make map modifiable

Set<Path> flowsPaths = flowsCode.keySet();
for (Path p: flowsPaths) {
logger.debug("Reading {}", p);
if (logger.isDebugEnabled()) {
logger.debug("Reading {}", p.getFileName());
}
flowsCode.put(p, Files.readString(p));
}

Expand Down Expand Up @@ -238,7 +257,8 @@ private Set<String> createFlows(Path dir, DeploymentDetails dd) throws IOExcepti
meta.setInputs(tresult.getInputs());
meta.setTimeout(tresult.getTimeout());
meta.setTimestamp(System.currentTimeMillis());
//No displayname, author or description. No handling of properties either
meta.setAuthor(dd.getProjectMetadata().getAuthor());
//No displayname or description. No handling of properties either

String compiled = tresult.getCode();
fl.setMetadata(meta);
Expand Down Expand Up @@ -285,22 +305,29 @@ private Set<String> createFlows(Path dir, DeploymentDetails dd) throws IOExcepti

}

private ZipFile compileAssetsArchive(Path root, Path webroot) throws IOException {
private ZipFile compileAssetsArchive(Path root, Path webroot, Path lib) throws IOException {

String rnd = rndName();

Path agama = Files.createDirectory(Paths.get(root.toString(), rnd));
logger.debug("Created temp directory {}", agama);
String agamStr = agama.toString();
logger.debug("Created temp directory");

Path ftl = Files.createDirectory(Paths.get(agamStr, "ftl"));
Path fl = Files.createDirectory(Paths.get(agamStr, "fl"));
Path scripts = Files.createDirectory(Paths.get(agamStr, SCRIPTS_SUBDIR));

Path ftl = Files.createDirectory(Paths.get(agama.toString(), "ftl"));
Path fl = Files.createDirectory(Paths.get(agama.toString(), "fl"));

logger.debug("Copying templates to {}", ftl);
Files.walkFileTree(webroot, copyVisitor(webroot, ftl, TEMPLATES_EXTENSIONS, true));
logger.debug("Copying assets to {}", fl);
Files.walkFileTree(webroot, copyVisitor(webroot, fl, TEMPLATES_EXTENSIONS, false));

//Make a zip with ftl and fl folders
if (Files.isDirectory(lib)) {
logger.debug("Copying .java and .groovy sources to {}", scripts);
Files.walkFileTree(lib, copyVisitor(lib, scripts, SCRIPTS_EXTENSIONS, true));
}

//Make a zip with scripts, ftl, and fl folders
ZipParameters params = new ZipParameters();
params.setCompressionMethod(CompressionMethod.STORE);

Expand All @@ -310,10 +337,55 @@ private ZipFile compileAssetsArchive(Path root, Path webroot) throws IOException
ZipFile newZip = new ZipFile(newZipPath.toFile());
newZip.addFolder(ftl.toFile(), params);
newZip.addFolder(fl.toFile(), params);
newZip.addFolder(scripts.toFile(), params);

return newZip;

}

private Set<String> computeSourcePaths(Path lib) throws IOException {

BiPredicate<Path, BasicFileAttributes> matcher = (path, attrs) -> attrs.isRegularFile() &&
Stream.of(SCRIPTS_EXTENSIONS).anyMatch(ext -> path.getFileName().toString().endsWith("." + ext));

if (Files.isDirectory(lib)) {
String slib = lib.toString();

try (Stream<Path> stream = Files.find(lib, 20, matcher)) {
return stream.map(Path::toString)
.map(s -> s.substring(slib.length() + 1)).collect(Collectors.toSet());
}
}
return Collections.emptySet();

}

private Set<String> transferJarFiles(Path lib) throws IOException {

Set<String> paths = new HashSet<>();
//All .jar files found at the top level are moved to the custom libs destination.
//This applies for VM-based installations only
if (!ON_CONTAINERS && Files.isDirectory(lib)) {
BiPredicate<Path, BasicFileAttributes> matcher =
(path, attrs) -> attrs.isRegularFile() && path.getFileName().toString().endsWith(".jar");

List<Path> list = null;
try (Stream<Path> stream = Files.find(lib, 1, matcher)) {
list = stream.collect(Collectors.toList());
}
logger.debug("Moving {} jar files to custom libs dir", list.size());

for (Path jar : list) {
String fn = jar.getFileName().toString();
paths.add(fn);

Files.move(jar, Paths.get(CUST_LIBS_DIR, fn), REPLACE_EXISTING);
logger.debug("{} moved", fn);
}
}
return paths;

}

private void updateFlowsAndAssets(List<Deployment> deployments) {

Expand All @@ -335,14 +407,16 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {

//If local map does not contain the given project or the local finishedAt value is less
//than the DB value, extract to disk the assets (including a previous directory purge)
//This conditional can only evaluate truthy in a multinode environment (containers) or
//upon application startup in a VM installation
if (finishedAt == null || finishedAt < d.getFinishedAt().getTime()) {
//Retrieve associated assets
String b64EncodedAssets = entryManager.find(d.getDn(), Deployment.class,
new String[]{ Deployment.ASSETS_ATTR }).getAssets();

try {
if (finishedAt != null) {
purge(projectsBasePaths.get(prjId));
purge(projectsBasePaths.get(prjId), projectsLibs.get(prjId));
}
if (b64EncodedAssets != null) {
extract(b64EncodedAssets, ASSETS_DIR);
Expand All @@ -369,7 +443,7 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {

try {
projectsFinishTimes.remove(prjId);
purge(projectsBasePaths.get(prjId));
purge(projectsBasePaths.get(prjId), projectsLibs.get(prjId));
} catch(IOException e) {
logger.error(e.getMessage());
}
Expand All @@ -378,20 +452,25 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {
}
}

Set<String> set;
projectsFlows.clear();
projectsBasePaths.clear();
projectsLibs.clear();
//Refresh maps wrt DB content
for (Deployment d : depls) {
String prjId = d.getId();
DeploymentDetails dd = d.getDetails();

set = Optional.ofNullable(d.getDetails().getFlowsError()).map(Map::keySet)
Set<String> set = Optional.ofNullable(dd.getFlowsError()).map(Map::keySet)
.orElse(new HashSet<>());
projectsFlows.put(prjId, set);

set = Optional.ofNullable(d.getDetails().getFolders()).map(HashSet::new)
set = Optional.ofNullable(dd.getFolders()).map(HashSet::new)
.orElse(new HashSet<>());
projectsBasePaths.put(prjId, set);

set = Optional.ofNullable(dd.getLibs()).map(HashSet::new)
.orElse(new HashSet<>());
projectsLibs.put(prjId, set);
}

}
Expand Down Expand Up @@ -439,20 +518,35 @@ private static String dnFromQname(String qname) {
// ========== File-system related utilities follow: ===========

//Walpurgis
private void purge(Set<String> dirs) throws IOException {
private void purge(Set<String> dirs, Set<String> filesToRemove) throws IOException {

if (dirs == null) return;

for (String dir : dirs) {
for (String subdir : ASSETS_SUBDIRS) {
Path p = Paths.get(ASSETS_DIR, subdir, dir);
if (dirs != null) {
for (String dir : dirs) {
for (String subdir : ASSETS_SUBDIRS) {
Path p = Paths.get(ASSETS_DIR, subdir, dir);

if (Files.isDirectory(p)) {
logger.info("Flushing folder {}", p);
removeDir(p);
if (Files.isDirectory(p)) {
logger.info("Flushing folder {}", p);
removeDir(p);
}
}
}
}

if (filesToRemove == null) return;

for (String f : filesToRemove) {
Path p = null;

if (f.endsWith(".jar")) {
p = Paths.get(CUST_LIBS_DIR, f);
} else {
p = Paths.get(ASSETS_DIR, SCRIPTS_SUBDIR, f);
}

logger.debug("Removing file {}", f);
Files.deleteIfExists(p);
}

}

Expand Down Expand Up @@ -487,10 +581,10 @@ private Path extractGamaFile(String b64EncodedContents) throws IOException {
}

private byte[] extractZipFileWithPurge(ZipFile zip, String destination,
Set<String> dirsPurge) throws IOException {
Set<String> dirsPurge, Set<String> filesToRemove) throws IOException {

Path zipPath = zip.getFile().toPath();
purge(dirsPurge);
purge(dirsPurge, filesToRemove);

logger.debug("Extracting contents of {} to {}", zipPath, destination);
zip.extractAll(destination);
Expand Down Expand Up @@ -577,6 +671,7 @@ private void init() {
projectsBasePaths = new HashMap<>();
projectsFinishTimes = new HashMap<>();
projectsFlows = new HashMap<>();
projectsLibs = new HashMap<>();

}

Expand Down
9 changes: 9 additions & 0 deletions jans-auth-server/agama/model/src/main/java/io/jans/ads/model/DeploymentDetails.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
public class DeploymentDetails {

private List<String> folders;
private List<String> libs;
private Map<String, String> flowsError;
private String error;
private ProjectMetadata metadata = new ProjectMetadata();
Expand All @@ -29,6 +30,14 @@ public void setFolders(List<String> folders) {
this.folders = folders;
}

public List<String> getLibs() {
return libs;
}

public void setLibs(List<String> libs) {
this.libs = libs;
}

public String getError() {
return error;
}
Expand Down

0 comments on commit 40268ad

Please sign in to comment.