Release 18.2.0

Release notes are available at: https://docs.dremio.com/release-notes/182-release-notes.html Change-Id: I968930a2cbfee960f6c827d0f4d1a1aa44763496
s1mbi0se · Oct 11, 2021 · 2c8cca2 · 2c8cca2
1 parent 6ec2ca5
commit 2c8cca2
Show file tree

Hide file tree

Showing 111 changed files with 222 additions and 197 deletions.
diff --git a/client/base/pom.xml b/client/base/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-client-base</artifactId>

diff --git a/client/jdbc/pom.xml b/client/jdbc/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.dremio.client</groupId>
     <artifactId>dremio-client-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
   <artifactId>dremio-client-jdbc</artifactId>
   <name>Client - JDBC Driver</name>

diff --git a/client/pom.xml b/client/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <groupId>com.dremio.client</groupId>

diff --git a/common/pom.xml b/common/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-common</artifactId>

diff --git a/connector/pom.xml b/connector/pom.xml
@@ -23,7 +23,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-connector</artifactId>

diff --git a/contrib/hive2-exec-shade/pom.xml b/contrib/hive2-exec-shade/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-hive2-exec-shaded</artifactId>

diff --git a/contrib/hive3-exec-shade/pom.xml b/contrib/hive3-exec-shade/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-hive3-exec-shaded</artifactId>

diff --git a/contrib/maprfs-shade/pom.xml b/contrib/maprfs-shade/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.contrib</groupId>
     <artifactId>dremio-contrib-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-maprfs-shaded</artifactId>

diff --git a/contrib/pom.xml b/contrib/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <groupId>com.dremio.contrib</groupId>

diff --git a/dac/backend/pom.xml b/dac/backend/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-dac-backend</artifactId>

diff --git a/dac/backend/src/main/java/com/dremio/dac/explore/join/JobsBasedRecommender.java b/dac/backend/src/main/java/com/dremio/dac/explore/join/JobsBasedRecommender.java
@@ -77,91 +77,96 @@ public JobsBasedRecommender(JobsService jobsService, NamespaceService namespaceS
 
   @Override
   public JoinRecommendations recommendJoins(Dataset dataset) {
-    final List<FieldOrigin> fieldOriginsList = listNotNull(dataset.getDatasetConfig().getFieldOriginsList());
-    if (fieldOriginsList.isEmpty()) {
-      // we can't help at this point
-      logger.warn("Could not find field origins in the provided dataset: " + dataset);
-      return new JoinRecommendations();
-    }
+    try {
+      final List<FieldOrigin> fieldOriginsList = listNotNull(dataset.getDatasetConfig().getFieldOriginsList());
+      if (fieldOriginsList.isEmpty()) {
+        // we can't help at this point
+        logger.warn("Could not find field origins in the provided dataset: " + dataset);
+        return new JoinRecommendations();
+      }
 
-    Set<List<String>> parents = new HashSet<>();
-    // index origin columns by their names
-    Map<Origin, String> refs = new HashMap<>();
-    indexOrigins(fieldOriginsList, parents, refs);
-
-    final long now = System.currentTimeMillis();
-    final List<JoinRecoForScoring> recommendations = new ArrayList<>();
-    // we cache namespace checks since we are very likely to see repeated datasets
-    final Map<NamespaceKey, Boolean> existsMap = new HashMap<>();
-
-    // 2nd step: for each parent table we look for existing joins.
-    for (List<String> parentDataset : parents) {
-
-      // find all jobs that refer to this parent
-      Iterable<JobDetails> jobsForParent = parentJobsProvider.getJobsForParent(parentDataset);
-      for (JobDetails jobDetails : jobsForParent) {
-        JobAttempt jobAttempt = JobsProtoUtil.getLastAttempt(jobDetails);
-        Long startTime = jobAttempt.getInfo().getStartTime();
-        //startTime of 0 means startTime was initially null
-        if (startTime == 0 || jobAttempt.getState() != COMPLETED) {
-          continue;
-        }
-        long recency = now - startTime;
-        JoinAnalysis joinAnalysis = jobAttempt.getInfo().getJoinAnalysis();
-
-        if (joinAnalysis != null && joinAnalysis.getJoinStatsList() != null && joinAnalysis.getJoinTablesList() != null) {
-          Map<Integer,JoinTable> joinTables = FluentIterable.from(joinAnalysis.getJoinTablesList())
-            .uniqueIndex(new Function<JoinTable, Integer>() {
-              @Override
-              public Integer apply(JoinTable joinTable) {
-                return joinTable.getTableId();
+      Set<List<String>> parents = new HashSet<>();
+      // index origin columns by their names
+      Map<Origin, String> refs = new HashMap<>();
+      indexOrigins(fieldOriginsList, parents, refs);
+
+      final long now = System.currentTimeMillis();
+      final List<JoinRecoForScoring> recommendations = new ArrayList<>();
+      // we cache namespace checks since we are very likely to see repeated datasets
+      final Map<NamespaceKey, Boolean> existsMap = new HashMap<>();
+
+      // 2nd step: for each parent table we look for existing joins.
+      for (List<String> parentDataset : parents) {
+
+        // find all jobs that refer to this parent
+        Iterable<JobDetails> jobsForParent = parentJobsProvider.getJobsForParent(parentDataset);
+        for (JobDetails jobDetails : jobsForParent) {
+          JobAttempt jobAttempt = JobsProtoUtil.getLastAttempt(jobDetails);
+          Long startTime = jobAttempt.getInfo().getStartTime();
+          //startTime of 0 means startTime was initially null
+          if (startTime == 0 || jobAttempt.getState() != COMPLETED) {
+            continue;
+          }
+          long recency = now - startTime;
+          JoinAnalysis joinAnalysis = jobAttempt.getInfo().getJoinAnalysis();
+
+          if (joinAnalysis != null && joinAnalysis.getJoinStatsList() != null && joinAnalysis.getJoinTablesList() != null) {
+            Map<Integer, JoinTable> joinTables = FluentIterable.from(joinAnalysis.getJoinTablesList())
+              .uniqueIndex(new Function<JoinTable, Integer>() {
+                @Override
+                public Integer apply(JoinTable joinTable) {
+                  return joinTable.getTableId();
+                }
+              });
+
+            for (final JoinStats join : joinAnalysis.getJoinStatsList()) {
+              // ignore if join analysis is missing join conditions
+              if (join.getJoinConditionsList() == null || join.getJoinConditionsList().isEmpty()) {
+                continue;
               }
-            });
+              final List<String> leftTablePathList = joinTables.get(join.getJoinConditionsList().get(0).getProbeSideTableId()).getTableSchemaPathList();
+              final List<String> rightTablePathList = joinTables.get(join.getJoinConditionsList().get(0).getBuildSideTableId()).getTableSchemaPathList();
 
-          for (final JoinStats join : joinAnalysis.getJoinStatsList()) {
-            // ignore if join analysis is missing join conditions
-            if (join.getJoinConditionsList() == null || join.getJoinConditionsList().isEmpty()) {
-              continue;
-            }
-            final List<String> leftTablePathList = joinTables.get(join.getJoinConditionsList().get(0).getProbeSideTableId()).getTableSchemaPathList();
-            final List<String> rightTablePathList = joinTables.get(join.getJoinConditionsList().get(0).getBuildSideTableId()).getTableSchemaPathList();
-
-            // if any of the join sides no longer exists in namespace, skip it
-            if (!checkIfExists(existsMap, leftTablePathList) || !checkIfExists(existsMap, rightTablePathList)) {
-              continue;
-            }
+              // if any of the join sides no longer exists in namespace, skip it
+              if (!checkIfExists(existsMap, leftTablePathList) || !checkIfExists(existsMap, rightTablePathList)) {
+                continue;
+              }
 
-            if (parents.contains(leftTablePathList)) {
-              addJoinReco(refs, recommendations, recency, join, rightTablePathList, leftTablePathList, joinTables);
-            }
-            // we can add it both ways if both tables are there
-            if (parents.contains(rightTablePathList)) {
-              addJoinReco(refs, recommendations, recency, join, leftTablePathList, rightTablePathList, joinTables);
+              if (parents.contains(leftTablePathList)) {
+                addJoinReco(refs, recommendations, recency, join, rightTablePathList, leftTablePathList, joinTables);
+              }
+              // we can add it both ways if both tables are there
+              if (parents.contains(rightTablePathList)) {
+                addJoinReco(refs, recommendations, recency, join, leftTablePathList, rightTablePathList, joinTables);
+              }
             }
           }
         }
       }
-    }
-    // sum up
+      // sum up
 
-    Builder<JoinRecommendation, JoinRecoForScoring> builder = ImmutableListMultimap.builder();
-    for (JoinRecoForScoring joinReco : recommendations) {
-      builder.put(joinReco.joinReco, joinReco);
-    }
-    ImmutableListMultimap<JoinRecommendation, JoinRecoForScoring> index = builder.build();
-    List<JoinRecoForScoring> mergedRecommendations = new ArrayList<>();
-    for (Entry<JoinRecommendation, Collection<JoinRecoForScoring>> recos : index.asMap().entrySet()) {
-      JoinRecommendation key = recos.getKey();
-      long recency = Long.MAX_VALUE;
-      int jobCount = 0;
-      for (JoinRecoForScoring joinReco : recos.getValue()) {
-        recency = Math.min(recency, joinReco.recency);
-        jobCount += joinReco.jobCount;
+      Builder<JoinRecommendation, JoinRecoForScoring> builder = ImmutableListMultimap.builder();
+      for (JoinRecoForScoring joinReco : recommendations) {
+        builder.put(joinReco.joinReco, joinReco);
+      }
+      ImmutableListMultimap<JoinRecommendation, JoinRecoForScoring> index = builder.build();
+      List<JoinRecoForScoring> mergedRecommendations = new ArrayList<>();
+      for (Entry<JoinRecommendation, Collection<JoinRecoForScoring>> recos : index.asMap().entrySet()) {
+        JoinRecommendation key = recos.getKey();
+        long recency = Long.MAX_VALUE;
+        int jobCount = 0;
+        for (JoinRecoForScoring joinReco : recos.getValue()) {
+          recency = Math.min(recency, joinReco.recency);
+          jobCount += joinReco.jobCount;
+        }
+        mergedRecommendations.add(new JoinRecoForScoring(key, jobCount, recency));
       }
-      mergedRecommendations.add(new JoinRecoForScoring(key, jobCount, recency));
+      Collections.sort(mergedRecommendations);
+      return recos(mergedRecommendations);
+    } catch (Exception e) {
+      logger.warn("Caught exception", e);
+      return new JoinRecommendations();
     }
-    Collections.sort(mergedRecommendations);
-    return recos(mergedRecommendations);
   }
 
   private boolean checkIfExists(final Map<NamespaceKey, Boolean> existsMap, List<String> path) {

diff --git a/dac/common/pom.xml b/dac/common/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-dac-common</artifactId>

diff --git a/dac/daemon/pom.xml b/dac/daemon/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-dac-daemon</artifactId>

diff --git a/dac/pom.xml b/dac/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-dac-parent</artifactId>

diff --git a/dac/ui-lib/pom.xml b/dac/ui-lib/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-ui-lib</artifactId>

diff --git a/dac/ui/pom.xml b/dac/ui/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-dac-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-dac-ui</artifactId>

diff --git a/dac/ui/src/components/DragComponents/ColumnMenuItem-spec.js b/dac/ui/src/components/DragComponents/ColumnMenuItem-spec.js
@@ -36,7 +36,7 @@ describe('ColumnMenuItem', () => {
   it('should render DragSource with preventDrag undefined, .column-draggable-icon if preventDrag is undefined', () => {
     const wrapper = shallow(<ColumnMenuItem {...minimalProps}/>);
     expect(wrapper.find(DragSource).prop('preventDrag')).to.be.undefined;
-    expect(wrapper.find({ class: 'column-draggable-icon' })).to.have.length(1);
+    expect(wrapper.find({ class: 'column-draggable-icon' })).to.have.length(0); // DX-37793: This currently built with wrong icon. Correct icon has not been designed yet.
   });
 
   it('should render DragSource with preventDrag true and not render .column-draggable-icon if preventDrag is true', () => {

diff --git a/dac/ui/src/components/DragComponents/ColumnMenuItem.js b/dac/ui/src/components/DragComponents/ColumnMenuItem.js
@@ -59,7 +59,7 @@ class ColumnMenuItem extends PureComponent {
     }
   }
 
-  renderDraggableIcon() {
+  renderDraggableIcon() { // DX-37793: This currently built with wrong icon. Correct icon has not been designed yet.
     return !this.props.preventDrag && !this.props.disabled ?
       (
         <div style={{position: 'relative', width: 0, height: 0}}>
@@ -132,7 +132,7 @@ class ColumnMenuItem extends PureComponent {
               // need to add a empty placeholder for partition icon to keep alignment
               item.get('isSorted') && !item.get('isPartitioned') && <div className={iconCls}></div>
             }
-            {this.renderDraggableIcon()}
+            {/* {this.renderDraggableIcon()} */} {/* DX-37793: This currently built with wrong icon. Correct icon has not been designed yet. */}
           </div>
         </DragSource>
       </div>

diff --git a/dac/ui/src/components/DragComponents/DragAreaColumn.js b/dac/ui/src/components/DragComponents/DragAreaColumn.js
@@ -226,11 +226,11 @@ export class DragAreaColumn extends Component {
       <div className={preventDrag ? disabledContent : content} key='custom-content'>
         <FontIcon type={typeToIconType[selectedColumn.get('type')]} key='custom-type' theme={styles.type}/>
         <EllipsedText style={styles.name} text={field.value} title={preventDrag ? formatMessage({ id: 'Read.Only'}) : field.value} />
-        {
+        {/* {
           !preventDrag ?
             <FontIcon type={'DropdownEnabled'} key='draggable' theme={styles.type}/>
             : null
-        }
+        } */}
       </div>
     );
   }

diff --git a/dac/ui/src/pages/ExplorePage/subpages/ExplorePageContentWrapper.js b/dac/ui/src/pages/ExplorePage/subpages/ExplorePageContentWrapper.js
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 import { PureComponent } from 'react';
-import Immutable from 'immutable';
+import Immutable, { Map } from 'immutable';
 import { connect } from 'react-redux';
 import PropTypes from 'prop-types';
 import classNames from 'classnames';
@@ -258,7 +258,7 @@ export default connect((state, { location }) => {
   // RBAC needs the permissions sent to the Acceleration components and passed down, in case any component along the way needs to be able to alter reflections
   const permissions = state.resources.entities.get('datasetUI')
   && state.resources.entities.get('datasetUI').first()
-  && state.resources.entities.get('datasetUI').first().get('permissions');
+  && (Map.isMap(state.resources.entities.get('datasetUI').first()) && state.resources.entities.get('datasetUI').first().get('permissions'));
   return ({
     canSelect: permissions && permissions.get('canSelect'),
     entityId: getDatasetEntityId(state, location),

diff --git a/distribution/jdbc-driver/pom.xml b/distribution/jdbc-driver/pom.xml
@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.dremio.distribution</groupId>
     <artifactId>dremio-distribution-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-jdbc-driver</artifactId>

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio</groupId>
     <artifactId>dremio-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <groupId>com.dremio.distribution</groupId>

diff --git a/distribution/resources/pom.xml b/distribution/resources/pom.xml
@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.dremio.distribution</groupId>
     <artifactId>dremio-distribution-parent</artifactId>
-    <version>18.1.0-202109222258120166-963adceb</version>
+    <version>18.2.0-202110081842420694-7e500f14</version>
   </parent>
 
   <artifactId>dremio-distribution-resources</artifactId>