Added form_key (optional string) to all Forms.

include/awkward/Content.h

@@ -20,6 +20,7 @@ namespace awkward {
   using ContentPtrVec = std::vector<std::shared_ptr<Content>>;
   class Form;
   using FormPtr       = std::shared_ptr<Form>;
+  using FormKey       = std::shared_ptr<std::string>;
 
   /// @class Form
   ///
@@ -38,9 +39,11 @@ namespace awkward {
     static FormPtr
       fromjson(const std::string& data);
 
-    /// @brief Called by subclass constructors; assigns #has_identities
-    /// and #parameters upon construction.
-    Form(bool has_identities, const util::Parameters& parameters);
+    /// @brief Called by subclass constructors; assigns #has_identities,
+    /// #parameters, and #form_key upon construction.
+    Form(bool has_identities,
+         const util::Parameters& parameters,
+         const FormKey& form_key);
 
     /// @brief Empty destructor; required for some C++ reason.
     virtual ~Form() { }
@@ -69,6 +72,8 @@ namespace awkward {
     /// #has_identities.
     /// @param check_parameters If `true`, Forms are not equal unless they have
     /// the same #parameters.
+    /// @param check_form_key If `true`, Forms are not equal unless they have
+    /// the same #form_key.
     /// @param compatibility_check If `true`, this is part of a compatibility
     /// check between an expected Form (`this`) and a generated array's Form
     /// (`other`). When the expected Form is a VirtualForm, it's allowed to be
@@ -77,8 +82,13 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const = 0;
 
+    /// @brief Returns `true` if this Form has the same #form_key as the other.
+    bool
+      form_key_equals(const FormKey& other_form_key) const;
+
     /// @brief The parameter associated with `key` at the first level
     /// that has a non-null value, descending only as deep as the first
     /// RecordForm.
@@ -111,6 +121,11 @@ namespace awkward {
     virtual int64_t
       purelist_depth() const = 0;
 
+    /// @brief An optional string associated with this Form, usually specifying
+    /// where an array may be fetched.
+    const FormKey
+      form_key() const;
+
     /// @brief Returns (a) the minimum list-depth and (b) the maximum
     /// list-depth of the array, which can differ if this array "branches"
     /// (differs when followed through different fields of a RecordForm or
@@ -192,21 +207,29 @@ namespace awkward {
 
     /// @brief Internal function for adding identities in #tojson.
     ///
-    /// Must be called between `builder.beginrecord()` and `builder.endrecord`.
+    /// Must be called between `builder.beginrecord()` and `builder.endrecord()`.
     void
       identities_tojson(ToJson& builder, bool verbose) const;
 
     /// @brief Internal function for adding parameters in #tojson.
     ///
-    /// Must be called between `builder.beginrecord()` and `builder.endrecord`.
+    /// Must be called between `builder.beginrecord()` and `builder.endrecord()`.
     void
       parameters_tojson(ToJson& builder, bool verbose) const;
 
+    /// @brief Internal function for adding form_key in #tojson.
+    ///
+    /// Must be called between `builder.beginrecord()` and `builder.endrecord()`.
+    void
+      form_key_tojson(ToJson& builder, bool verbose) const;
+
     protected:
-    /// @brief See #has_identities;
+    /// @brief See #has_identities
     bool has_identities_;
-    /// @brief See #parameters;
+    /// @brief See #parameters
     util::Parameters parameters_;
+    /// @brief See #form_key
+    FormKey form_key_;
   };
 
   /// @class Content

include/awkward/array/BitMaskedArray.h

@@ -26,6 +26,7 @@ namespace awkward {
     /// @brief Creates a BitMaskedForm. See BitMaskedArray for documentation.
     BitMaskedForm(bool has_identities,
                   const util::Parameters& parameters,
+                  const FormKey& form_key,
                   Index::Form mask,
                   const FormPtr& content,
                   bool valid_when,
@@ -86,6 +87,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/ByteMaskedArray.h

@@ -25,6 +25,7 @@ namespace awkward {
     /// @brief Creates a ByteMaskedForm. See ByteMaskedArray for documentation.
     ByteMaskedForm(bool has_identities,
                    const util::Parameters& parameters,
+                   const FormKey& form_key,
                    Index::Form mask,
                    const FormPtr& content,
                    bool valid_when);
@@ -81,6 +82,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/EmptyArray.h

@@ -21,7 +21,8 @@ namespace awkward {
   public:
     /// @brief Creates a EmptyForm. See EmptyArray for documentation.
     EmptyForm(bool has_identities,
-              const util::Parameters& parameters);
+              const util::Parameters& parameters,
+              const FormKey& form_key);
 
     const TypePtr
       type(const util::TypeStrs& typestrs) const override;
@@ -66,6 +67,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
   };
 

include/awkward/array/IndexedArray.h

@@ -23,6 +23,7 @@ namespace awkward {
     /// for documentation.
     IndexedForm(bool has_identities,
                 const util::Parameters& parameters,
+                const FormKey& form_key,
                 Index::Form index,
                 const FormPtr& content);
 
@@ -75,6 +76,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:
@@ -90,6 +92,7 @@ namespace awkward {
     /// @brief Creates a IndexedOptionForm. See IndexedArray for documentation.
     IndexedOptionForm(bool has_identities,
                       const util::Parameters& parameters,
+                      const FormKey& form_key,
                       Index::Form index,
                       const FormPtr& content);
 
@@ -140,6 +143,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/ListArray.h

@@ -21,6 +21,7 @@ namespace awkward {
     /// documentation.
     ListForm(bool has_identities,
              const util::Parameters& parameters,
+             const FormKey& form_key,
              Index::Form starts,
              Index::Form stops,
              const FormPtr& content);
@@ -77,6 +78,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/ListOffsetArray.h

@@ -21,6 +21,7 @@ namespace awkward {
     /// {@link ListOffsetArrayOf ListOffsetArray} for documentation.
     ListOffsetForm(bool has_identities,
                    const util::Parameters& parameters,
+                   const FormKey& form_key,
                    Index::Form offsets,
                    const FormPtr& content);
 
@@ -73,6 +74,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/NumpyArray.h

@@ -23,6 +23,7 @@ namespace awkward {
     /// @brief Creates a NumpyForm. See NumpyArray for documentation.
     NumpyForm(bool has_identities,
               const util::Parameters& parameters,
+              const FormKey& form_key,
               const std::vector<int64_t>& inner_shape,
               int64_t itemsize,
               const std::string& format,
@@ -95,6 +96,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/RawArray.h

@@ -42,8 +42,9 @@ namespace awkward {
     /// @brief Creates a RawForm. See RawArray for documentation.
     RawForm(bool has_identities,
             const util::Parameters& parameters,
+            const FormKey& form_key,
             const std::string& T)
-        : Form(has_identities, parameters)
+        : Form(has_identities, parameters, form_key)
         , T_(T) { }
 
     const std::string
@@ -63,6 +64,7 @@ namespace awkward {
       shallow_copy() const override {
       return std::make_shared<RawForm>(has_identities_,
                                        parameters_,
+                                       form_key_,
                                        T_);
     }
 
@@ -123,6 +125,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override {
       throw std::runtime_error("FIXME: RawForm::equal");
     }
@@ -403,6 +406,7 @@ namespace awkward {
       form(bool materialize) const override {
       return std::make_shared<RawForm>(identities_.get() != nullptr,
                                        parameters_,
+                                       FormKey(nullptr),
                                        typeid(T).name());
     }
 

include/awkward/array/RecordArray.h

@@ -21,6 +21,7 @@ namespace awkward {
     /// documentation.
     RecordForm(bool has_identities,
                const util::Parameters& parameters,
+               const FormKey& form_key,
                const util::RecordLookupPtr& recordlookup,
                const std::vector<FormPtr>& contents);
 
@@ -85,6 +86,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/RegularArray.h

@@ -20,6 +20,7 @@ namespace awkward {
     /// @brief Creates a RegularForm. See RegularArray for documentation.
     RegularForm(bool has_identities,
                 const util::Parameters& parameters,
+                const FormKey& form_key,
                 const FormPtr& content,
                 int64_t size);
 
@@ -72,6 +73,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/UnionArray.h

@@ -23,6 +23,7 @@ namespace awkward {
     /// documentation.
     UnionForm(bool has_identities,
               const util::Parameters& parameters,
+              const FormKey& form_key,
               Index::Form tags,
               Index::Form index,
               const std::vector<FormPtr>& contents);
@@ -85,6 +86,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/UnmaskedArray.h

@@ -21,6 +21,7 @@ namespace awkward {
     /// @brief Creates a UnmaskedForm. See UnmaskedArray for documentation.
     UnmaskedForm(bool has_identities,
                  const util::Parameters& parameters,
+                 const FormKey& form_key,
                  const FormPtr& content);
 
     const FormPtr
@@ -69,6 +70,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

include/awkward/array/VirtualArray.h

@@ -22,6 +22,7 @@ namespace awkward {
     /// @brief Creates a VirtualForm. See VirtualArray for documentation.
     VirtualForm(bool has_identities,
                 const util::Parameters& parameters,
+                const FormKey& form_key,
                 const FormPtr& form,
                 bool has_length);
 
@@ -77,6 +78,7 @@ namespace awkward {
       equal(const FormPtr& other,
             bool check_identities,
             bool check_parameters,
+            bool check_form_key,
             bool compatibility_check) const override;
 
   private:

src/awkward1/highlevel.py

@@ -1341,6 +1341,29 @@ def numba_type(self):
 
         return numba.typeof(self._numbaview)
 
+    def __getstate__(self):
+        form, container, num_partitions = awkward1.to_arrayset(self)
+        if self._behavior is awkward1.behavior:
+            behavior = None
+        else:
+            behavior = self._behavior
+        if self._metadata is None:
+            metadata = None
+        else:
+            metadata = dict(self._metadata)
+        return form, container, num_partitions, behavior, metadata
+
+    def __setstate__(self, state):
+        form, container, num_partitions, behavior, metadata = state
+        layout = awkward1.from_arrayset(
+            form, container, num_partitions, highlevel=False
+        )
+        if self.__class__ is Array:
+            self.__class__ = awkward1._util.arrayclass(layout, behavior)
+        self.layout = layout
+        self.behavior = behavior
+        self.metadata = metadata
+
 
 class Record(awkward1._connect._numpy.NDArrayOperatorsMixin):
     """
@@ -1882,6 +1905,30 @@ def numba_type(self):
 
         return numba.typeof(self._numbaview)
 
+    def __getstate__(self):
+        form, container, num_partitions = awkward1.to_arrayset(self._layout.array)
+        if self._behavior is awkward1.behavior:
+            behavior = None
+        else:
+            behavior = self._behavior
+        if self._metadata is None:
+            metadata = None
+        else:
+            metadata = dict(self._metadata)
+        return form, container, num_partitions, behavior, metadata, self._layout.at
+
+    def __setstate__(self, state):
+        form, container, num_partitions, behavior, metadata, at = state
+        array = awkward1.from_arrayset(
+            form, container, num_partitions, highlevel=False
+        )
+        layout = awkward1.layout.Record(array, at)
+        if self.__class__ is Record:
+            self.__class__ = awkward1._util.recordclass(layout, behavior)
+        self.layout = layout
+        self.behavior = behavior
+        self.metadata = metadata
+
 
 class ArrayBuilder(object):
     """