Simplifying primitives.

primitives/test_primitives/increment.py

@@ -88,33 +88,10 @@ class IncrementPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, H
         # If "inputs" is container.DataFrame, then also result is.
         outputs = inputs + self.hyperparams['amount']
 
-        # In the case of container.DataFrame, operations metadata is not preserved.
-        # We update with required metadata. "source" tells which primitive generated this metadata.
-        outputs.metadata = outputs.metadata.update((), {
-            'schema': metadata_base.CONTAINER_SCHEMA_VERSION,
-            'structural_type': type(outputs),
-        }, source=self)
-
-        # Now we describe a shape.
-        dimension_index = None
-        for dimension_index, dimension_length in enumerate(outputs.shape):
-            outputs.metadata = outputs.metadata.update((metadata_base.ALL_ELEMENTS,) * dimension_index, {
-                'dimension': {
-                    'length': dimension_length,
-                }
-            }, source=self)
-
-        assert dimension_index is not None
-
-        if len(set(inputs.dtypes)) == 1:
-            outputs.metadata = outputs.metadata.update((metadata_base.ALL_ELEMENTS,) * (dimension_index + 1), {
-                'structural_type': inputs.dtypes[0].type,
-            }, source=self)
-        else:
-            for i, dtype in enumerate(outputs.dtypes):
-                outputs.metadata = outputs.metadata.update(typing.cast(metadata_base.Selector, (metadata_base.ALL_ELEMENTS,) * dimension_index) + typing.Cast(metadata_base.Selector, (i,)), {
-                    'structural_type': dtype.type,
-                }, source=self)
+        # Metadata might not be preserved through operations, so we make sure and update metadata ourselves.
+        # Because just values changed (but not structure) and the primitive is a transformation, we can reuse
+        # inputs metadata, but just update the value reference. "source" tells which primitive generated this metadata.
+        outputs.metadata = inputs.metadata.set_for_value(outputs, source=self)
 
         # Wrap it into default "CallResult" object: we are not doing any iterations.
         return base.CallResult(outputs)
@@ -135,6 +112,8 @@ class IncrementPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, H
             return output_metadata
 
         inputs_metadata = typing.cast(metadata_base.DataMetadata, arguments['inputs'])
+
+        # Try to get structural types defined for all elements.
         dimension_index = 0
         while True:
             metadata = inputs_metadata.query((metadata_base.ALL_ELEMENTS,) * dimension_index)
@@ -148,6 +127,7 @@ class IncrementPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, H
 
         if inputs_value_structural_type is None:
             # TODO: Check if every element individually is a numeric type.
+            #       There was no structural type defined for all elements, but there should be one for each column or even each element.
             return None
 
         # Not a perfect way to check for a numeric type but will do for this example.

primitives/test_primitives/monomial.py

 import os.path
-import typing
 
 from d3m import container, utils
 from d3m.metadata import hyperparams, base as metadata_base, params
@@ -93,21 +92,12 @@ class MonomialPrimitive(supervised_learning.SupervisedLearnerPrimitiveBase[Input
         # We convert a regular list to container list which supports metadata attribute.
         outputs: container.List[float] = container.List[float](result)
 
-        # We clear old metadata (but which keeps history and link to inputs metadata) and set new metadata.
-        # "for_value" tells that this new metadata will be associated with "outputs",
-        # and "source" tells which primitive generated this metadata.
-        metadata = inputs.metadata.clear({
-            'schema': metadata_base.CONTAINER_SCHEMA_VERSION,
-            'structural_type': type(outputs),
-            'dimension': {
-                'length': len(outputs)
-            }
-        }, for_value=outputs, source=self).update((metadata_base.ALL_ELEMENTS,), {
-            'structural_type': float,
-        }, source=self)
-
-        # Set metadata attribute.
-        outputs.metadata = metadata
+        # Even if the structure of outputs is the same as inputs, conceptually, outputs
+        # are different, they are new data. So we do not reuse metadata from inputs but create
+        # new metadata. We do this by clearing old metadata which keeps history and link the
+        # to inputs metadata. "for_value" tells that this new metadata will be associated with
+        # "outputs" and "source" tells which primitive generated this metadata.
+        outputs.metadata = inputs.metadata.clear(for_value=outputs, source=self)
 
         # Wrap it into default "CallResult" object: we are not doing any iterations.
         return base.CallResult(outputs)

primitives/test_primitives/random.py

@@ -90,21 +90,11 @@ class RandomPrimitive(generator.GeneratorPrimitiveBase[Outputs, None, Hyperparam
         # We convert a regular ndarray to a container DataFrame which supports metadata attribute.
         outputs = container.DataFrame(result)
 
-        # We clear old metadata (but which keeps history and link to inputs metadata) and set new metadata.
-        # "for_value" tells that this new metadata will be associated with "outputs",
-        # and "source" tells which primitive generated this metadata.
-        metadata = inputs.metadata.clear({
-            'schema': metadata_base.CONTAINER_SCHEMA_VERSION,
-            'structural_type': type(outputs),
-            'dimension': {
-                'length': len(outputs)
-            }
-        }, for_value=outputs, source=self).update((metadata_base.ALL_ELEMENTS,), {
-            'structural_type': float,
-        }, source=self)
-
-        # Set metadata attribute.
-        outputs.metadata = metadata
+        # Outputs are different from inputs, so we do not reuse metadata from inputs but create
+        # new metadata. We do this by clearing old metadata which keeps history and link the
+        # to inputs metadata. "for_value" tells that this new metadata will be associated with
+        # "outputs" and "source" tells which primitive generated this metadata.
+        outputs.metadata = inputs.metadata.clear(for_value=outputs, source=self)
 
         # Wrap it into default "CallResult" object: we are not doing any iterations.
         return base.CallResult(outputs)

primitives/test_primitives/sum.py

@@ -141,18 +141,15 @@ class SumPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperpa
         if response.status != 200:
             raise ValueError("Invalid HTTP response status: {status}".format(status=response.status))
 
-        result = int(response.read())
-
-        outputs = container.List[float]((result,), {
-            'schema': metadata_base.CONTAINER_SCHEMA_VERSION,
-            'structural_type': container.List[float],
-            'dimension': {
-                'length': 1,
-            },
-        })
-        outputs.metadata = outputs.metadata.update((metadata_base.ALL_ELEMENTS,), {
-            'structural_type': float,
-        })
+        result = float(response.read())
+
+        outputs = container.List[float]((result,))
+
+        # Outputs are different from inputs, so we do not reuse metadata from inputs but create
+        # new metadata. We do this by clearing old metadata which keeps history and link the
+        # to inputs metadata. "for_value" tells that this new metadata will be associated with
+        # "outputs" and "source" tells which primitive generated this metadata.
+        outputs.metadata = inputs.metadata.clear(for_value=outputs, source=self)
 
         # Wrap it into default "CallResult" object: we are not doing any iterations.
         return base.CallResult(outputs)
@@ -173,6 +170,8 @@ class SumPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperpa
             return output_metadata
 
         inputs_metadata = typing.cast(metadata_base.DataMetadata, arguments['inputs'])
+
+        # Try to get structural types defined for all elements.
         dimension_index = 0
         while True:
             metadata = inputs_metadata.query((metadata_base.ALL_ELEMENTS,) * dimension_index)
@@ -186,6 +185,7 @@ class SumPrimitive(transformer.TransformerPrimitiveBase[Inputs, Outputs, Hyperpa
 
         if inputs_value_structural_type is None:
             # TODO: Check if every element individually is a numeric type.
+            #       There was no structural type defined for all elements, but there should be one for each element.
             return None
 
         # Not a perfect way to check for a numeric type but will do for this example.