Revert "HADOOP-19180. EC: Fix calculation errors caused by special index order (#6813). Contributed by zhengchenyu."

This reverts commit e5b76dc.

Author: zhengchenyu

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java

@@ -51,6 +51,7 @@ public class RSRawDecoder extends RawErasureDecoder {
   private byte[] gfTables;
   private int[] cachedErasedIndexes;
   private int[] validIndexes;
+  private int numErasedDataUnits;
   private boolean[] erasureFlags;
 
   public RSRawDecoder(ErasureCoderOptions coderOptions) {
@@ -119,10 +120,14 @@ private void processErasures(int[] erasedIndexes) {
     this.gfTables = new byte[getNumAllUnits() * getNumDataUnits() * 32];
 
     this.erasureFlags = new boolean[getNumAllUnits()];
+    this.numErasedDataUnits = 0;
 
     for (int i = 0; i < erasedIndexes.length; i++) {
       int index = erasedIndexes[i];
       erasureFlags[index] = true;
+      if (index < getNumDataUnits()) {
+        numErasedDataUnits++;
+      }
     }
 
     generateDecodeMatrix(erasedIndexes);
@@ -151,22 +156,21 @@ private void generateDecodeMatrix(int[] erasedIndexes) {
 
     GF256.gfInvertMatrix(tmpMatrix, invertMatrix, getNumDataUnits());
 
-    for (p = 0; p < erasedIndexes.length; p++) {
-      int erasedIndex = erasedIndexes[p];
-      if (erasedIndex < getNumDataUnits()) {
+    for (i = 0; i < numErasedDataUnits; i++) {
+      for (j = 0; j < getNumDataUnits(); j++) {
+        decodeMatrix[getNumDataUnits() * i + j] =
+                invertMatrix[getNumDataUnits() * erasedIndexes[i] + j];
+      }
+    }
+
+    for (p = numErasedDataUnits; p < erasedIndexes.length; p++) {
+      for (i = 0; i < getNumDataUnits(); i++) {
+        s = 0;
         for (j = 0; j < getNumDataUnits(); j++) {
-          decodeMatrix[getNumDataUnits() * p + j] =
-              invertMatrix[getNumDataUnits() * erasedIndexes[p] + j];
-        }
-      } else {
-        for (i = 0; i < getNumDataUnits(); i++) {
-          s = 0;
-          for (j = 0; j < getNumDataUnits(); j++) {
-            s ^= GF256.gfMul(invertMatrix[j * getNumDataUnits() + i],
-                encodeMatrix[getNumDataUnits() * erasedIndexes[p] + j]);
-          }
-          decodeMatrix[getNumDataUnits() * p + i] = s;
+          s ^= GF256.gfMul(invertMatrix[j * getNumDataUnits() + i],
+                  encodeMatrix[getNumDataUnits() * erasedIndexes[p] + j]);
         }
+        decodeMatrix[getNumDataUnits() * p + i] = s;
       }
     }
   }

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.c

@@ -132,6 +132,9 @@ static int processErasures(IsalDecoder* pCoder, unsigned char** inputs,
     index = erasedIndexes[i];
     pCoder->erasedIndexes[i] = index;
     pCoder->erasureFlags[index] = 1;
+    if (index < numDataUnits) {
+      pCoder->numErasedDataUnits++;
+    }
   }
 
   pCoder->numErased = numErased;
@@ -172,6 +175,7 @@ int decode(IsalDecoder* pCoder, unsigned char** inputs,
 
 // Clear variables used per decode call
 void clearDecoder(IsalDecoder* decoder) {
+  decoder->numErasedDataUnits = 0;
   decoder->numErased = 0;
   memset(decoder->gftbls, 0, sizeof(decoder->gftbls));
   memset(decoder->decodeMatrix, 0, sizeof(decoder->decodeMatrix));
@@ -201,24 +205,24 @@ int generateDecodeMatrix(IsalDecoder* pCoder) {
   h_gf_invert_matrix(pCoder->tmpMatrix,
                                 pCoder->invertMatrix, numDataUnits);
 
-  for (p = 0; p < pCoder->numErased; p++) {
+  for (i = 0; i < pCoder->numErasedDataUnits; i++) {
     for (j = 0; j < numDataUnits; j++) {
-      int erasedIndex = pCoder->erasedIndexes[p];
-      if (erasedIndex < numDataUnits) {
-        pCoder->decodeMatrix[numDataUnits * p + j] =
-            pCoder->invertMatrix[numDataUnits *
-                                     pCoder->erasedIndexes[p] + j];
-      } else {
-        for (i = 0; i < numDataUnits; i++) {
-          s = 0;
-          for (j = 0; j < numDataUnits; j++) {
-            s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i],
-                          pCoder->encodeMatrix[numDataUnits *
-                            pCoder->erasedIndexes[p] + j]);
-          }
-          pCoder->decodeMatrix[numDataUnits * p + i] = s;
-        }
+      pCoder->decodeMatrix[numDataUnits * i + j] =
+                      pCoder->invertMatrix[numDataUnits *
+                      pCoder->erasedIndexes[i] + j];
+    }
+  }
+
+  for (p = pCoder->numErasedDataUnits; p < pCoder->numErased; p++) {
+    for (i = 0; i < numDataUnits; i++) {
+      s = 0;
+      for (j = 0; j < numDataUnits; j++) {
+        s ^= h_gf_mul(pCoder->invertMatrix[j * numDataUnits + i],
+          pCoder->encodeMatrix[numDataUnits *
+                                        pCoder->erasedIndexes[p] + j]);
       }
+
+      pCoder->decodeMatrix[numDataUnits * p + i] = s;
     }
   }
 

hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/erasurecode/erasure_coder.h

@@ -62,6 +62,7 @@ typedef struct _IsalDecoder {
   unsigned char erasureFlags[MMAX];
   int erasedIndexes[MMAX];
   int numErased;
+  int numErasedDataUnits;
   unsigned char* realInputs[MMAX];
 } IsalDecoder;
 

hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/io/erasurecode/erasure_code_test.c

@@ -27,27 +27,25 @@
 #include "erasure_code.h"
 #include "gf_util.h"
 #include "erasure_coder.h"
-#include "dump.h"
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 int main(int argc, char *argv[]) {
-  int i, j, k, l;
+  int i, j;
   char err[256];
   size_t err_len = sizeof(err);
   int chunkSize = 1024;
   int numDataUnits = 6;
   int numParityUnits = 3;
-  int numTotalUnits = numDataUnits + numParityUnits;
   unsigned char** dataUnits;
   unsigned char** parityUnits;
   IsalEncoder* pEncoder;
-  int erasedIndexes[3];
+  int erasedIndexes[2];
   unsigned char* allUnits[MMAX];
   IsalDecoder* pDecoder;
-  unsigned char* decodingOutput[3];
+  unsigned char* decodingOutput[2];
   unsigned char** backupUnits;
 
   if (0 == build_support_erasurecode()) {
@@ -84,11 +82,6 @@ int main(int argc, char *argv[]) {
     }
   }
 
-  // Allocate decode output
-  for (i = 0; i < 3; i++) {
-    decodingOutput[i] = malloc(chunkSize);
-  }
-
   pEncoder = (IsalEncoder*)malloc(sizeof(IsalEncoder));
   memset(pEncoder, 0, sizeof(*pEncoder));
   initEncoder(pEncoder, numDataUnits, numParityUnits);
@@ -102,53 +95,26 @@ int main(int argc, char *argv[]) {
   memcpy(allUnits + numDataUnits, parityUnits,
                             numParityUnits * (sizeof (unsigned char*)));
 
-  for (i = 0; i < numTotalUnits; i++) {
-    for (j = 0; j < numTotalUnits; j++) {
-      for (k = 0; k < numTotalUnits; k++) {
-        int numErased;
-        if (i == j && j == k) {
-          erasedIndexes[0] = i;
-          numErased = 1;
-          backupUnits[0] = allUnits[i];
-          allUnits[i] = NULL;
-        } else if (i == j) {
-          erasedIndexes[0] = i;
-          erasedIndexes[1] = k;
-          numErased = 2;
-          backupUnits[0] = allUnits[i];
-          backupUnits[1] = allUnits[k];
-          allUnits[i] = NULL;
-          allUnits[k] = NULL;
-        } else if (i == k || j == k) {
-          erasedIndexes[0] = i;
-          erasedIndexes[1] = j;
-          numErased = 2;
-          backupUnits[0] = allUnits[i];
-          backupUnits[1] = allUnits[j];
-          allUnits[i] = NULL;
-          allUnits[j] = NULL;
-        } else {
-          erasedIndexes[0] = i;
-          erasedIndexes[1] = j;
-          erasedIndexes[2] = k;
-          numErased = 3;
-          backupUnits[0] = allUnits[i];
-          backupUnits[1] = allUnits[j];
-          backupUnits[2] = allUnits[k];
-          allUnits[i] = NULL;
-          allUnits[j] = NULL;
-          allUnits[k] = NULL;
-        }
-        decode(pDecoder, allUnits, erasedIndexes, numErased, decodingOutput, chunkSize);
-        for (l = 0; l < pDecoder->numErased; l++) {
-          if (0 != memcmp(decodingOutput[l], backupUnits[l], chunkSize)) {
-            printf("Decoding failed\n");
-            dumpDecoder(pDecoder);
-            return -1;
-          }
-          allUnits[erasedIndexes[l]] = backupUnits[l];
-        }
-      }
+  erasedIndexes[0] = 1;
+  erasedIndexes[1] = 7;
+
+  backupUnits[0] = allUnits[1];
+  backupUnits[1] = allUnits[7];
+
+  allUnits[0] = NULL; // Not to read
+  allUnits[1] = NULL;
+  allUnits[7] = NULL;
+
+  decodingOutput[0] = malloc(chunkSize);
+  decodingOutput[1] = malloc(chunkSize);
+
+  decode(pDecoder, allUnits, erasedIndexes, 2, decodingOutput, chunkSize);
+
+  for (i = 0; i < pDecoder->numErased; i++) {
+    if (0 != memcmp(decodingOutput[i], backupUnits[i], chunkSize)) {
+      fprintf(stderr, "Decoding failed\n\n");
+      dumpDecoder(pDecoder);
+      return -1;
     }
   }