Commit eb17cf12 authored by Mitar's avatar Mitar

Merge branch 'no-enumerations' into 'devel'

No enumerations

Closes #133 and #132

See merge request !167
parents 71bf9013 f7ed79db
Pipeline #131563079 passed with stage
in 4 minutes and 9 seconds
## vNEXT
* Converted all enumerations to strings. Added fields to `HelloResponse`
for TA2 to be able to communicate supported enumeration values.
[#132](https://gitlab.com/datadrivendiscovery/ta3ta2-api/-/issues/132)
[#133](https://gitlab.com/datadrivendiscovery/ta3ta2-api/-/issues/133)
[!167](https://gitlab.com/datadrivendiscovery/ta3ta2-api/-/merge_requests/167)
* Added `use_default_values_for_free_hyperparams` flag to `SearchSolutionsRequest`
to precisely control the behavior when a search template contains no placeholders
but free hyper-parameters.
......
......@@ -19,36 +19,33 @@ extend google.protobuf.FileOptions {
// and "SearchSolutionsResponse" messages.
option (protocol_version) = "2020.2.12_pre";
enum EvaluationMethod {
// Default value. Not to be used.
EVALUATION_METHOD_UNDEFINED = 0;
// The following are the only evaluation methods required
// to be supported for the "ScoreSolution" call.
HOLDOUT = 1;
K_FOLD = 2;
// This evaluation method can be used to ask TA2 to rank a
// solution as part of all found solutions of a given
// "SearchSolutions" call. Invalid to use before the
// "SearchSolutions" call, of which the solution is part of,
// has successfully completed or has been stopped.
// Only possible with "RANK" performance metric.
RANKING = 99;
// The rest are defined to allow expressing internal evaluation
// methods used by TA2 during solution search. If any method being used
// is missing, feel free to request it to be added.
LEAVE_ONE_OUT = 100;
// Instead of really scoring, a TA2 might predict the score only.
PREDICTION = 101;
// Training data is reused to test as well.
TRAINING_DATA = 102;
}
message ScoringConfiguration {
// The evaluation method to use.
EvaluationMethod method = 1;
// The evaluation method to use. Standard values are required to
// be supported and are:
// * "HOLDOUT"
// * "K_FOLD"
//
// In addition, "RANKING" evaluation method should be also supported.
// "RANKING" evaluation method can be used for a rank of a solution
// among all found solutions of a given "SearchSolutions" call.
// Invalid to use before the "SearchSolutions" call, of which the
// solution is part of, has successfully completed or has been stopped.
// Only possible with "RANK" performance metric.
//
// The following additional evaluation methods are defined to allow
// expressing internal evaluation methods used by TA2 during solution
// search. If any method being used is missing, feel free to request
// it to be added.
// * "LEAVE_ONE_OUT"
// * "PREDICTION"
// Instead of really scoring, a TA2 might predict the score only.
// * "TRAINING_DATA"
// Training data is reused to test as well.
//
// No standard value will ever start with "_", so if you worry about
// potential conflicts with future standard values, consider starting
// your non-standard values with "_'.
string method = 1;
// Number of folds made, if applicable.
int32 folds = 2;
// If applicable, the ratio between the train and test data and represents the proportion of
......@@ -135,7 +132,7 @@ message SearchSolutionsRequest {
// Which value types can a TA2 system use to communicate values to a TA3 system?
// The order is important as a TA2 system will try value types in order until one works out,
// or an error will be returned instead of the value.
repeated ValueType allowed_value_types = 5;
repeated string allowed_value_types = 5;
// Problem description to use for the solution search. Problem description is optional
// with fully specified pipelines which TA3 provides. If still provided, TA2 should use
// it to set semantic types on target columns and any other use of a problem description
......@@ -390,7 +387,7 @@ message FitSolutionRequest {
// The order is important as TA2 system will try value types in order until one works out,
// or an error will be returned instead of the value. An error exposing a value does not
// stop the overall process.
repeated ValueType expose_value_types = 4;
repeated string expose_value_types = 4;
// Any users associated with this call itself. Optional.
repeated SolutionRunUser users = 5;
// Seed for pseudorandom number generator. Note, not setting this value
......@@ -438,7 +435,7 @@ message ProduceSolutionRequest {
// The order is important as the TA2 system will try value types in order until one works
// out, or an error will be returned instead of the value. An error exposing a value does
// not stop the overall process.
repeated ValueType expose_value_types = 4;
repeated string expose_value_types = 4;
// Any users associated with this call itself. Optional.
repeated SolutionRunUser users = 5;
}
......@@ -499,13 +496,13 @@ message DataAvailableResponse {}
message SplitDataRequest {
// The input dataset to be split.
Value input = 1;
// How to perform the split. Note that only HOLDOUT and K_FOLD make sense for the
// How to perform the split. Note that only "HOLDOUT" and "K_FOLD" make sense for the
// EvaluationMethod.
ScoringConfiguration scoring_configuration = 2;
// Which value types should be used for outputs.
// The order is important as the TA2 system will try value types in order until one works
// out, or an error will be returned instead of the value.
repeated ValueType allowed_value_types = 3;
repeated string allowed_value_types = 3;
// The problem, which is necessary for some splitting methods like stratified splits.
ProblemDescription problem = 4;
}
......@@ -607,9 +604,12 @@ message HelloResponse {
// List of value types that a TA3 system can use to communicate values to a TA2 system.
// The order is important as a TA3 system should try value types in order until one works
// out, or an error will be returned instead of the value.
repeated ValueType allowed_value_types = 3;
repeated string allowed_value_types = 3;
// List of API extensions that a TA2 supports.
repeated string supported_extensions = 4;
repeated string supported_task_keywords = 5;
repeated string supported_performance_metrics = 6;
repeated string supported_evaluation_methods = 7;
}
// See each message's comments for information about each particular call.
......
......@@ -98,15 +98,6 @@ message PipelineSource {
repeated PipelineDescription pipelines = 3;
}
// TODO: Deprecated. Remove.
enum PipelineContext {
PIPELINE_CONTEXT_UNKNOWN = 0;
PRETRAINING = 1;
TESTING = 2;
EVALUATION = 3;
PRODUCTION = 4;
}
// User associated with the creation of the template/pipeline, or selection of a primitive.
message PipelineDescriptionUser {
// Globally unique ID for this user. It can be opaque, but it should identify the same user
......@@ -204,8 +195,6 @@ message PipelineDescription {
// TA2: Timestamp when created. Templates do not have this timestamp. TA3 might provide it for
// a fully specified pipeline.
google.protobuf.Timestamp created = 3;
// TODO: Deprecated. Remove.
PipelineContext context = 4 [deprecated=true];
// Human friendly name of the pipeline. For templates it can be a hint to
// TA2 how to name found pipelines. Optional.
string name = 5;
......
syntax = "proto3";
option go_package = "pipeline";
// Task keyword of the problem.
enum TaskKeyword {
// Default value. Not to be used.
TASK_KEYWORD_UNDEFINED = 0;
CLASSIFICATION = 1;
REGRESSION = 2;
CLUSTERING = 3;
LINK_PREDICTION = 4;
VERTEX_NOMINATION = 5;
VERTEX_CLASSIFICATION = 6;
COMMUNITY_DETECTION = 7;
GRAPH_MATCHING = 8;
FORECASTING = 9;
COLLABORATIVE_FILTERING = 10;
OBJECT_DETECTION = 11;
SEMISUPERVISED = 12;
BINARY = 13;
MULTICLASS = 14;
MULTILABEL = 15;
UNIVARIATE = 16;
MULTIVARIATE = 17;
OVERLAPPING = 18;
NONOVERLAPPING = 19;
TABULAR = 20;
RELATIONAL = 21;
IMAGE = 22;
AUDIO = 23;
VIDEO = 24;
SPEECH = 25;
TEXT = 26;
GRAPH = 27;
MULTIGRAPH = 28;
TIME_SERIES = 29;
GROUPED = 30;
GEOSPATIAL = 31;
REMOTE_SENSING = 32;
LUPI = 33;
MISSING_METADATA = 34;
}
// The evaluation metric for any potential solution.
enum PerformanceMetric {
// Default value. Not to be used.
METRIC_UNDEFINED = 0;
// The following are the only evaluation methods required
// to be supported for the ScoreSolution call.
ACCURACY = 1;
PRECISION = 2;
RECALL = 3;
F1 = 4;
F1_MICRO = 5;
F1_MACRO = 6;
ROC_AUC = 7;
ROC_AUC_MICRO = 8;
ROC_AUC_MACRO = 9;
MEAN_SQUARED_ERROR = 10;
ROOT_MEAN_SQUARED_ERROR = 11;
MEAN_ABSOLUTE_ERROR = 12;
R_SQUARED = 13;
NORMALIZED_MUTUAL_INFORMATION = 14;
JACCARD_SIMILARITY_SCORE = 15;
PRECISION_AT_TOP_K = 16;
OBJECT_DETECTION_AVERAGE_PRECISION = 17;
HAMMING_LOSS = 18;
// This metric can be used to ask TA2 to rank a solution as part of
// all found solutions of a given "SearchSolutions" call. Rank is a
// floating-point number. Lower numbers represent better solutions.
// Presently evaluation requirements are that ranks should be non-negative
// and that each ranked pipeline have a different rank (for all
// solutions of a given SearchSolutions call). Only possible with
// "RANKING" evaluation method.
RANK = 99;
// The rest are defined to allow expressing internal evaluation
// scores used by TA2 during pipeline search. If any you are using
// is missing, feel free to request it to be added.
// Average loss of an unspecified loss function.
LOSS = 100;
}
message ProblemPerformanceMetric {
PerformanceMetric metric = 1;
// The evaluation metric. Standard values are required to be supported
// and are available here:
// https://metadata.datadrivendiscovery.org/devel/?definitions#definitions.performance_metric
//
// In addition, "RANK" metric should be also supported. "RANK" metric
// can be used for a rank of a solution among all found solutions of
// a given "SearchSolutions" call. Rank is a non-negative floating-point number.
// Lower numbers represent better solutions. Each ranked solution should have
// a different rank (for all solutions of a given "SearchSolutions" call).
// Only possible with "RANKING" evaluation method.
//
// The following additional metrics are defined to allow expressing
// internal evaluation scores used by TA2 during solution search.
// If any you are using is missing, feel free to request it to be added.
// * "LOSS"
// Average loss of an unspecified loss function.
//
// No standard value will ever start with "_", so if you worry about
// potential conflicts with future standard values, consider starting
// your non-standard values with "_'.
string metric = 1;
// Additional params used by some metrics.
int32 k = 2;
string pos_label = 3;
}
message Problem {
repeated TaskKeyword task_keywords = 8;
// Task keywords of the problem. Standard values are required
// to be supported and are available here:
// https://metadata.datadrivendiscovery.org/devel/?definitions#definitions.problem.task_keywords
//
// No standard value will ever start with "_", so if you worry about
// potential conflicts with future standard values, consider starting
// your non-standard values with "_'.
repeated string task_keywords = 8;
repeated ProblemPerformanceMetric performance_metrics = 7;
}
......
......@@ -65,17 +65,17 @@ class ValueType(d3m_utils.Enum):
"""
Enumeration of possible value types.
Values are kept in sync with TA2-TA3 API's ``ValueType`` enumeration.
Values are kept in sync with standard TA2-TA3 API's value types.
"""
RAW = 1
DATASET_URI = 2
CSV_URI = 3
PICKLE_URI = 4
PICKLE_BLOB = 5
PLASMA_ID = 6
LARGE_RAW = 7
LARGE_PICKLE_BLOB = 8
RAW = 'RAW'
DATASET_URI = 'DATASET_URI'
CSV_URI = 'CSV_URI'
PICKLE_URI = 'PICKLE_URI'
PICKLE_BLOB = 'PICKLE_BLOB'
PLASMA_ID = 'PLASMA_ID'
LARGE_RAW = 'LARGE_RAW'
LARGE_PICKLE_BLOB = 'LARGE_PICKLE_BLOB'
def _can_encode_raw(value):
......@@ -957,7 +957,7 @@ def decode_performance_metric(metric):
try:
metric_value = problem_module.PerformanceMetric(metric.metric)
except ValueError:
metric_value = problem_pb2.PerformanceMetric.Name(metric.metric)
metric_value = metric.metric
decoded_metric = {
'metric': metric_value,
......
......@@ -6,9 +6,9 @@ import tempfile
import unittest
from d3m import container, exceptions, index, utils as d3m_utils
from d3m.metadata import pipeline as pipeline_module, problem as problem_module, base as metadata_base
from d3m.metadata import pipeline as pipeline_module, problem as problem_module
from ta3ta2_api import utils, pipeline_pb2, problem_pb2
from ta3ta2_api import utils, problem_pb2
TEST_PRIMITIVES_DIR = os.path.join(os.path.dirname(__file__), 'data', 'primitives')
TEST_PROBLEMS_DIR = os.path.join(os.path.dirname(__file__), 'data', 'problems')
......@@ -202,7 +202,7 @@ class TestUtils(unittest.TestCase):
'metric': 'RANK'
})
self.assertEqual(grpc_message.metric, problem_pb2.PerformanceMetric.Value('RANK'))
self.assertEqual(grpc_message.metric, 'RANK')
metric = utils.decode_performance_metric(grpc_message)
......@@ -213,13 +213,13 @@ class TestUtils(unittest.TestCase):
'metric': 'F1_MICRO'
})
self.assertEqual(grpc_message.metric, problem_pb2.PerformanceMetric.Value('F1_MICRO'))
self.assertEqual(grpc_message.metric, 'F1_MICRO')
grpc_message = utils.encode_performance_metric({
'metric': problem_module.PerformanceMetric.F1_MICRO
})
self.assertEqual(grpc_message.metric, problem_pb2.PerformanceMetric.Value('F1_MICRO'))
self.assertEqual(grpc_message.metric, 'F1_MICRO')
metric = utils.decode_performance_metric(grpc_message)
......@@ -243,46 +243,5 @@ class TestUtils(unittest.TestCase):
utils.validate_uri('file:///output/datasetDoc.json', ['/output/'])
class TestProto(unittest.TestCase):
def assert_enums_match(self, d3m_enum, grpc_enum):
# Convert D3M enum to dict
d3m_values = {
e.name: e.value
for e in d3m_enum
}
# Convert gRPC enum to dict
grpc_values = dict(grpc_enum.items())
# Check the '_UNDEFINED' element, discard it
undefined, = [k for k, v in grpc_values.items() if v == 0]
self.assertTrue(undefined.endswith(('_UNDEFINED', '_UNKNOWN')))
grpc_values.pop(undefined)
# Also ignore values above 99, which are private to gRPC
grpc_values = {k: v for k, v in grpc_values.items() if v < 99}
# Check they match
self.assertEqual(d3m_values, grpc_values)
def test_performance_metric(self):
self.assert_enums_match(
problem_module.PerformanceMetric,
problem_pb2.PerformanceMetric,
)
def test_task_keywords(self):
self.assert_enums_match(
problem_module.TaskKeyword,
problem_pb2.TaskKeyword,
)
def test_pipeline_context(self):
self.assert_enums_match(
metadata_base.Context,
pipeline_pb2.PipelineContext,
)
if __name__ == '__main__':
unittest.main()
......@@ -19,43 +19,40 @@ option go_package = "pipeline";
// provided to the other system, the list is traversed in order and the first
// value type which can be used without an error is used. If the list is
// exhausted, then an error is provided instead.
enum ValueType {
// Default value. Not to be used.
VALUE_TYPE_UNDEFINED = 0;
// The following value types are those everyone should support.
// Raw value. Not all values can be represented as a raw value.
// The value before encoding should be at most 64 KB.
RAW = 1;
// Represent the value as a D3M dataset. Only "file://" schema is supported using a
// shared file system. Dataset URI should point to the "datasetDoc.json" file of the dataset.
// Only Dataset container values can be represented this way.
DATASET_URI = 2;
// Represent the value as a CSV file. Only "file://" schema is supported using a
// shared file system. CSV URI should point to the file with ".csv" file extension.
// Only tabular container values with numeric and string cell values can be represented
// this way.
CSV_URI = 3;
// The following are additional value types which can be supported by systems,
// but it is not required. If the value cannot be represented with value types your system
// supports and your system is still asked to do so, it should return "ValueError" error instead.
// Represent values by Python-pickling them. Only "file://" schema is supported using a
// shared file system. Pickle URI should point to the file with ".pickle" file extension.
PICKLE_URI = 4;
// Represent values by Python-pickling them but sending them through the API.
// The value before encoding should be at most 64 KB.
PICKLE_BLOB = 5;
// Represent values with arrow and storing them into shared instance of Plasma.
PLASMA_ID = 6;
// Same as "RAW", but without any size limit.
LARGE_RAW = 7;
// Same as "PICKLE_BLOB", but without any size limit.
LARGE_PICKLE_BLOB = 8;
}
//
// The following value types are those everyone should support.
// * "RAW"
// Raw value. Not all values can be represented as a raw value.
// The value before encoding should be at most 64 KB.
// * "DATASET_URI"
// Represent the value as a D3M dataset. Only "file://" schema is supported using a
// shared file system. Dataset URI should point to the "datasetDoc.json" file of the dataset.
// Only Dataset container values can be represented this way.
// * "CSV_URI"
// Represent the value as a CSV file. Only "file://" schema is supported using a
// shared file system. CSV URI should point to the file with ".csv" file extension.
// Only tabular container values with numeric and string cell values can be represented
// this way.
//
// The following are additional value types which can be supported by systems,
// but it is not required. If the value cannot be represented with value types your system
// supports and your system is still asked to do so, it should return "ValueError" error instead.
// * "PICKLE_URI"
// Represent values by Python-pickling them. Only "file://" schema is supported using a
// shared file system. Pickle URI should point to the file with ".pickle" file extension.
// * "PICKLE_BLOB"
// Represent values by Python-pickling them but sending them through the API.
// The value before encoding should be at most 64 KB.
// * "PLASMA_ID"
// Represent values with arrow and storing them into shared instance of Plasma.
// * "LARGE_RAW"
// Same as "RAW", but without any size limit.
// * "LARGE_PICKLE_BLOB"
// Same as "PICKLE_BLOB", but without any size limit.
//
// No standard value will ever start with "_", so if you worry about
// potential conflicts with future standard values, consider starting
// your non-standard values with "_'.
message ValueError {
// A error message useful for debugging or logging. Not meant to be very end-user friendly.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment