WX-915 Restore Job Lists Page (#780)

* wx-922 saving current progress, linter working on vscode

* WX-922 updated circle jobs, added scripts to package.json, disabled certain linting rules for now

* WX-922 added tsc job, added ES2022 lib for compilation

* WX-922 testing if tsc and eslint jobs are configured

* wx-922 corrected pkg manager version to that referenced in repo

* wx-723 initial setup of oidc library, checking to see if setup works

* WX-723 library up and running, need to figure out oauth uri configurations

* wx-723 redirect works, user profile being read in, need to come up with prototype silent refresh flow

* wx-723 code cleanup, moving to implement silent refresh

* wx-723 pulled async methods out of synchronous constructors

* wx-723 updated tests, running async auth service method at the start of component ngInit, updated tsconfig settings for test runners, still working on silent refresh

* wx-723 some code cleanup

* wx-723 switched to localStorage, updates to redirect logic

* wx-723 code corrections to storage and redirect method call

* wx-723 removed silent refresh files, code cleanup

* wx-723 removed unused static files and scripts

* wx-723 added tests for redirect component, syntax corrections

* wx-723 updated README.md

* wx-915 restored job list page, working through pagination issues on both front and back-ends

* wx-915 fixed encode/decode logic for jobs page, updated default redirect to point to jobs page

* wx-915 updated decode tests so that incoming tokens are strings

* wx-915 more updates to tests to ensure that strings, not bytes, are being supplied as decode arg parameters

* wx-915 removed comment

* wx-915 added conditionals to get_response_message method

* wx-915 updating tests (one more test left to fix)

* wx-915 updated table column test

* wx-915 restored back to jobs button and associated tests

* wx-915 restored redirect from root to job list

* Update TERRA_QUICKSTART.md

* wx-915 fixed pagination button styling and missing column selection

* wx-915 updated removed fuction reference in mat-chip components

* wx-915 updated test for filter chip

* wx-915 added front end solution for display value updates, css update for table count box

* wx-915 matched table count border color with pagination buttons' border values

---------

Co-authored-by: Janet Gainer-Dewar <jdewar@broadinstitute.org>

TERRA_QUICKSTART.md

@@ -56,15 +56,14 @@ work the same as local testing.
 ## Releasing JobManager in Terra
 
 Start by following release instructions in [the README](README.md#build-docker-images-and-releases).
-Once the new version exists in GCR and Github, update the dev environment and BEE template in Beehive. 
-For each of these:
- * broad.io/beehive/r/chart-release/swatomation/jobmanager
- * broad.io/beehive/r/chart-release/dev/jobmanager
-Follow these steps:
+Once the new version exists in GCR and Github, update the dev environment in Beehive. 
+[Go here](broad.io/beehive/r/chart-release/dev/jobmanager) and follow these steps:
  * Click on `On Demand Deployment > Change Versions`
- * In the `Specify App Version > Set Exact Version` box, change the version to the one you just pushed
+ * In the `Specify App Version > Set Exact Version` box, change the version to the one you just pushed. 
+ Make sure you include the EXACT tag you pushed with, it's easy to forget the `v`.
  * Click `Calculate and Preview`
  * After reviewing changes, click `Apply 1 change` on the right
-In dev, this will kick off a Github Action that will update the dev k8s cluster.
+This will kick off a Github Action that will update the dev k8s cluster. Test in dev and ensure that JM is functional
+and your changes are present.
 
 The version should automatically roll out to prod with the new monolith release.

servers/cromwell/jobs/controllers/jobs_controller.py

@@ -114,7 +114,6 @@ def get_job(id, **kwargs):
 
     :rtype: JobMetadataResponse
     """
-
     url = '{cromwell_url}/{id}/metadata?{query}'.format(
         cromwell_url=_get_base_url(),
         id=id,
@@ -456,7 +455,6 @@ def query_jobs(body, **kwargs):
     page = page_from_offset(offset, query_page_size)
 
     has_auth = headers is not None
-
     response = requests.post(_get_base_url() + '/query',
                              json=cromwell_query_params(
                                  query, page, query_page_size, has_auth),
@@ -477,7 +475,7 @@ def query_jobs(body, **kwargs):
     next_page_token = page_tokens.encode_offset(offset + query_page_size)
     return QueryJobsResponse(results=jobs_list,
                              total_size=total_results,
-                             next_page_token=next_page_token)
+                             next_page_token=next_page_token.decode())
 
 
 def get_last_page(total_results, page_size):
@@ -596,7 +594,6 @@ def get_operation_details(job, operation, **kwargs):
 
         if response.status_code != 200:
             handle_error(response)
-
         return JobOperationResponse(id=job, details=response.text)
 
 
@@ -758,8 +755,11 @@ def _is_call_cached(metadata):
 
 def _get_response_message(response):
     logger.error('Response error: {}'.format(response.content))
-    if is_jsonable(response) and response.json().get('message'):
-        return response.json().get('message')
+    if is_jsonable(response):
+        try:
+            return response.json().get('message')
+        except:
+            return str(response.json())
     return str(response)
 
 

servers/jm_utils/jm_utils/page_tokens.py

@@ -31,15 +31,15 @@ def _decode(token):
     """Decodes the pagination token.
 
     Args:
-        token: (string) Base64 encoded JSON pagination token
+        token: (string) JSON pagination token
 
     Returns:
         (dict) The token dictionary representing a page of items.
     """
     if token is None:
         return None
     # Pad the token out to be divisible by 4.
-    padded_token = token + '='.encode() * (4 - (len(token) % 4))
+    padded_token = bytes(token, 'utf8') + '='.encode() * (4 - (len(token) % 4))
     decoded_token = base64.urlsafe_b64decode(padded_token)
     token_dict = json.loads(decoded_token)
     if not token_dict or not isinstance(token_dict, dict):

servers/jm_utils/jm_utils/test/test_page_tokens.py

@@ -14,7 +14,7 @@ class TestJmUtils(unittest.TestCase):
     """ jm_utils unit tests """
 
     def test_encode_decode_offset(self):
-        encoded = page_tokens.encode_offset(12)
+        encoded = page_tokens.encode_offset(12).decode()
         decoded = page_tokens.decode_offset(encoded)
         self.assertEqual(decoded, 12)
 
@@ -25,9 +25,9 @@ def test_encode_offset_zero(self):
                       str(context.exception))
 
     def test_decode_offset_zero(self):
-        encoded = page_tokens._encode({'of': 0})
+        token = page_tokens._encode({'of': 0}).decode()
         with self.assertRaises(ValueError) as context:
-            page_tokens.decode_offset(encoded)
+            page_tokens.decode_offset(token)
         self.assertIn('Invalid offset token JSON', str(context.exception))
 
     def test_decode_offset_none(self):
@@ -36,7 +36,7 @@ def test_decode_offset_none(self):
     def test_encode_decode_create_time_max(self):
         now = datetime.datetime.now().replace(microsecond=0).replace(
             tzinfo=pytz.utc)
-        encoded = page_tokens.encode_create_time_max(now, 'offset-id')
+        encoded = page_tokens.encode_create_time_max(now, 'offset-id').decode()
         decoded_create_time, decoded_offset_id = page_tokens.decode_create_time_max(
             encoded)
         self.assertEqual(decoded_create_time, now)
@@ -53,14 +53,14 @@ def test_encode_create_time_max_invalid(self):
                       str(context.exception))
 
     def test_decode_create_time_max_invalid(self):
-        encoded = page_tokens._encode({"cb": "not-a-date"})
+        token = page_tokens._encode({"cb": "not-a-date"}).decode()
         with self.assertRaises(ValueError) as context:
-            page_tokens.decode_create_time_max(encoded)
+            page_tokens.decode_create_time_max(token)
         self.assertIn("Invalid created before in token JSON",
                       str(context.exception))
-        encoded = page_tokens._encode({'cb': 10, 'oi': 123})
+        token = page_tokens._encode({'cb': 10, 'oi': 123}).decode()
         with self.assertRaises(ValueError) as context:
-            page_tokens.decode_create_time_max(encoded)
+            page_tokens.decode_create_time_max(token)
         self.assertIn("Invalid offset ID in token JSON",
                       str(context.exception))
 

ui/src/app/app-routing.module.ts

@@ -24,49 +24,54 @@ import { SignInRedirectComponent } from './sign-in/sign-in-redirect.component';
 // UI Components to the <router-outlet> element in the main AppComponent.
 const routes: Routes = [
   {
-    path: '',
-    component: PagenotfoundComponent
+    path: "",
+    redirectTo: "jobs",
+    pathMatch: "full",
   },
   {
-    path: 'sign_in',
+    path: "sign_in",
     component: SignInComponent,
-    canActivate: [CapabilitiesActivator]
+    canActivate: [CapabilitiesActivator],
   },
   {
-    path: 'projects',
+    path: "projects",
     component: ProjectsComponent,
-    canActivate: [CapabilitiesActivator]
+    canActivate: [CapabilitiesActivator],
   },
   {
-    path: 'dashboard',
+    path: "dashboard",
     component: DashboardComponent,
     //TODO: (zach) if the projectId param is missing, it gives a 400 error which is not desired.
     // Should be redirect to the home page.
     canActivate: [CapabilitiesActivator],
-    runGuardsAndResolvers: 'always',
+    runGuardsAndResolvers: "always",
     resolve: {
-      aggregations: DashboardResolver
-    }
+      aggregations: DashboardResolver,
+    },
   },
   {
-    path: 'jobs',
-    component: PagenotfoundComponent
+    path: "jobs",
+    component: JobListComponent,
+    canActivate: [CapabilitiesActivator],
+    resolve: {
+      stream: JobListResolver,
+    },
   },
   {
-    path: 'jobs/:id',
+    path: "jobs/:id",
     component: JobDetailsComponent,
     canActivate: [CapabilitiesActivator],
     resolve: {
-      job: JobDetailsResolver
-    }
+      job: JobDetailsResolver,
+    },
   },
   {
-    path: 'redirect-from-oauth',
-    component: SignInRedirectComponent
+    path: "redirect-from-oauth",
+    component: SignInRedirectComponent,
   },
   {
-    path: '**',
-    component: PagenotfoundComponent
+    path: "**",
+    component: PagenotfoundComponent,
   },
 ];
 

ui/src/app/job-details/job-details.component.html

@@ -1,6 +1,7 @@
 <div class="details-container">
   <jm-panels [job]="job"
              [primaryLabels]="primaryLabels"
+             (close)="handleClose()"
              (navUp)="handleNavUp()"></jm-panels>
   <jm-resources *ngIf="hasResources()" [job]="job"></jm-resources>
   <!-- Only show the tasks table if there is at least 1 task. -->

ui/src/app/job-details/job-details.component.spec.ts

@@ -141,6 +141,27 @@ describe('JobDetailsComponent', () => {
     expect(de.query(By.css('#job-id')).nativeElement.value).toContain(jobId);
   }));
 
+  it("navigates to jobs table on close", fakeAsync(() => {
+    const q = URLSearchParamsUtils.encodeURLSearchParams({
+      extensions: { projectId: "foo" },
+    });
+    router.navigate(["jobs/" + jobId], { queryParams: { q } });
+    tick();
+
+    fixture.detectChanges();
+    tick();
+
+    const de: DebugElement = fixture.debugElement;
+    de.query(By.css(".close")).nativeElement.click();
+    fixture.detectChanges();
+    tick();
+
+    const fakeJobs = fixture.debugElement.query(By.css(".fake-jobs"));
+    expect(fakeJobs).toBeTruthy();
+    const fakeJobsRoute = fakeJobs.componentInstance.route;
+    expect(fakeJobsRoute.snapshot.queryParams["q"]).toEqual(q);
+  }));
+
   @Component({
     selector: 'jm-test-app',
     template: '<router-outlet></router-outlet>'

ui/src/app/job-details/job-details.component.ts

@@ -12,9 +12,9 @@ import {CapabilitiesResponse} from "../shared/model/CapabilitiesResponse";
 import {objectNotEmpty} from '../shared/common';
 
 @Component({
-  selector: 'jm-job-details',
-  templateUrl: './job-details.component.html',
-  styleUrls: ['./job-details.component.css'],
+  selector: "jm-job-details",
+  templateUrl: "./job-details.component.html",
+  styleUrls: ["./job-details.component.css"],
 })
 export class JobDetailsComponent implements OnInit {
   @ViewChild(JobTabsComponent) taskTabs;
@@ -29,29 +29,47 @@ export class JobDetailsComponent implements OnInit {
     private readonly router: Router,
     private readonly route: ActivatedRoute,
     private readonly settingsService: SettingsService,
-    private readonly capabilitiesService: CapabilitiesService) {
+    private readonly capabilitiesService: CapabilitiesService
+  ) {
     this.capabilities = capabilitiesService.getCapabilitiesSynchronous();
   }
 
   ngOnInit(): void {
-    this.job = this.route.snapshot.data['job'];
-    const req = URLSearchParamsUtils.unpackURLSearchParams(this.route.snapshot.queryParams['q']);
-    this.projectId = req.extensions.projectId || '';
+    this.job = this.route.snapshot.data["job"];
+    const req = URLSearchParamsUtils.unpackURLSearchParams(
+      this.route.snapshot.queryParams["q"]
+    );
+    this.projectId = req.extensions.projectId || "";
 
     // if the user has saved settings for display columns, use that
     // otherwise, go with default list from capabilities
-    if (this.settingsService.getSavedSettingValue('displayColumns', this.projectId)) {
-      this.primaryLabels = this.settingsService.getSavedSettingValue('displayColumns', this.projectId).filter(field => field.match('labels.')).map(field => field.replace('labels.',''));
+    if (
+      this.settingsService.getSavedSettingValue(
+        "displayColumns",
+        this.projectId
+      )
+    ) {
+      this.primaryLabels = this.settingsService
+        .getSavedSettingValue("displayColumns", this.projectId)
+        .filter((field) => field.match("labels."))
+        .map((field) => field.replace("labels.", ""));
     } else if (this.capabilities.displayFields) {
-      this.primaryLabels = this.capabilities.displayFields.map((df) => df.field).filter(field => field.match('labels.')).map(field => field.replace('labels.',''));
+      this.primaryLabels = this.capabilities.displayFields
+        .map((df) => df.field)
+        .filter((field) => field.match("labels."))
+        .map((field) => field.replace("labels.", ""));
     } else if (this.job.labels) {
       this.primaryLabels = Object.keys(this.job.labels);
     }
   }
 
   hasTabs(): boolean {
-    if (objectNotEmpty(this.job.inputs) || objectNotEmpty(this.job.outputs) || objectNotEmpty(this.job.failures)) {
-       return true;
+    if (
+      objectNotEmpty(this.job.inputs) ||
+      objectNotEmpty(this.job.outputs) ||
+      objectNotEmpty(this.job.failures)
+    ) {
+      return true;
     }
     if (this.job.extensions) {
       let tasks: TaskMetadata[] = this.job.extensions.tasks || [];
@@ -61,45 +79,61 @@ export class JobDetailsComponent implements OnInit {
 
   handleNavUp(): void {
     if (this.job.extensions.parentJobId) {
-      this.router.navigate(['/jobs/' + this.job.extensions.parentJobId], {
+      this.router
+        .navigate(["/jobs/" + this.job.extensions.parentJobId], {
+          queryParams: {
+            q: this.route.snapshot.queryParams["q"],
+          },
+          replaceUrl: true,
+          skipLocationChange: false,
+        })
+        .then((result) => {
+          this.handleNav();
+        });
+    }
+  }
+
+  handleNavDown(id: string): void {
+    this.router
+      .navigate(["/jobs/" + id], {
         queryParams: {
-          'q': this.route.snapshot.queryParams['q']
+          q: this.route.snapshot.queryParams["q"],
         },
         replaceUrl: true,
-        skipLocationChange: false
+        skipLocationChange: false,
       })
-      .then(result => {
+      .then((result) => {
         this.handleNav();
       });
-    }
   }
 
-  handleNavDown(id: string): void {
-    this.router.navigate(['/jobs/' + id], {
+  handleClose(): void {
+    this.router.navigate(["jobs"], {
       queryParams: {
-        'q': this.route.snapshot.queryParams['q']
+        q: this.route.snapshot.queryParams["q"],
       },
-      replaceUrl: true,
-      skipLocationChange: false
-    })
-    .then(result => {
-      this.handleNav();
     });
   }
 
   hasResources(): boolean {
-    return (this.job.extensions && (this.job.extensions.sourceFile || this.job.extensions.logs));
+    return (
+      this.job.extensions &&
+      (this.job.extensions.sourceFile || this.job.extensions.logs)
+    );
   }
 
   private handleNav() {
-    this.job = this.route.snapshot.data['job'];
+    this.job = this.route.snapshot.data["job"];
     this.jobPanels.job = this.job;
     this.jobPanels.setUpExtensions();
     if (this.taskTabs.failuresTable) {
       this.taskTabs.failuresTable.dataSource = this.job.failures;
     }
     if (this.jobPanels.jobFailures) {
-      this.jobPanels.jobFailures.dataSource = this.job.failures.slice(0, this.jobPanels.numOfErrorsToShow);
+      this.jobPanels.jobFailures.dataSource = this.job.failures.slice(
+        0,
+        this.jobPanels.numOfErrorsToShow
+      );
     }
     if (this.job.extensions.tasks) {
       this.taskTabs.timingDiagram.buildTimelineData(this.job.extensions.tasks);