Commit 35c21f4a authored by jun liu's avatar jun liu Committed by Sujen

Isi v2019.6.7 primitive and sample pipelines final update

parent 91036888
......@@ -22,7 +22,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"schema": "https://metadata.datadrivendiscovery.org/schemas/v0/primitive.json",
......@@ -185,5 +185,5 @@
}
},
"structural_type": "dsbox.datapostprocessing.ensemble_voting.EnsembleVoting",
"digest": "2cb4d81bfe64821ca655d329cc8fff7e8054764eecf5979ac7e34e689ba778c3"
"digest": "f9ec132819a2daebddc99a2fecac011a89b0e88c14ea7173801b797bd9efca49"
}
{
"id":"6d5af388-e635-483f-9427-515e5e9e7476",
"schema":"https://metadata.datadrivendiscovery.org/schemas/v0/pipeline.json",
"created":"2019-06-10T22:36:16.413877Z",
"inputs":[
{
"name":"input dataset"
}
],
"outputs":[
{
"data":"steps.6.produce",
"name":"predictions of input dataset"
}
],
"steps":[
{
"type":"PRIMITIVE",
"primitive":{
"id":"f31f8c1f-d1c5-43e5-a4b2-2ae4a761ef2e",
"version":"0.2.0",
"python_path":"d3m.primitives.data_transformation.denormalize.Common",
"name":"Denormalize datasets",
"digest":"948abcebe21aa6765ecc8caf73c314058a623be2665cf69eb57c7d7662db23c7"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"inputs.0"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"starting_resource":{
"type":"VALUE",
"data":null
},
"recursive":{
"type":"VALUE",
"data":true
},
"many_to_many":{
"type":"VALUE",
"data":false
},
"discard_not_joined_tabular_resources":{
"type":"VALUE",
"data":false
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"4b42ce1e-9b98-4a25-b68e-fad13311eb65",
"version":"0.3.0",
"python_path":"d3m.primitives.data_transformation.dataset_to_dataframe.Common",
"name":"Extract a DataFrame from a Dataset",
"digest":"9935952c663b051a9a5bf25045adb3ce7803b724f8397306dabd458a3350627d"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.0.produce"
}
},
"outputs":[
{
"id":"produce"
}
]
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version":"0.2.0",
"python_path":"d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name":"Extracts columns by semantic type",
"digest":"9111567d84de0ffe913eee93395a4a3f1eceb2d8db600b39523257297f09a181"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.1.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"semantic_types":{
"type":"VALUE",
"data":[
"https://metadata.datadrivendiscovery.org/types/PrimaryKey",
"https://metadata.datadrivendiscovery.org/types/FileName"
]
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version":"0.2.0",
"python_path":"d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name":"Extracts columns by semantic type",
"digest":"9111567d84de0ffe913eee93395a4a3f1eceb2d8db600b39523257297f09a181"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.1.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"semantic_types":{
"type":"VALUE",
"data":[
"https://metadata.datadrivendiscovery.org/types/TrueTarget"
]
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"a29b0080-aeff-407d-9edb-0aa3eefbde01",
"version":"0.2.0",
"python_path":"d3m.primitives.data_preprocessing.video_reader.DataFrameCommon",
"name":"Columns video reader",
"digest":"67a102c1fb435cab3b2ddbe3e568d3835c09b01cee877791a123ce105dcb8c4f"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.2.produce"
}
},
"outputs":[
{
"id":"produce"
}
]
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"dsbox-featurizer-image-inceptionV3",
"version":"1.5.2",
"python_path":"d3m.primitives.feature_extraction.inceptionV3_image_feature.DSBOX",
"name":"DSBox Image Featurizer inceptionV3",
"digest":"6622a86ce5c6462cd7a6a7b258e369e22bf692754f7b1dc16a24d1aa4758af1a"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.4.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"use_limitation":{
"type":"VALUE",
"data":false
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"dsbox-featurizer-video-classification-lstm",
"version":"1.5.2",
"python_path":"d3m.primitives.classification.lstm.DSBOX",
"name":"DSBox Video Classification LSTM",
"digest":"a275f2d85b93df3a1b85339ca6fe2916e346724af3dac4ad1ad57a53f823367a"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.5.produce"
},
"outputs":{
"type":"CONTAINER",
"data":"steps.3.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"LSTM_units":{
"type":"VALUE",
"data":1024
},
"epochs":{
"type":"VALUE",
"data":100
}
}
}
],
"name":"DefaultVideoClassificationTemplate:5273959808",
"description":"",
"digest":"dc104869b2f7d2b489e2d088640b7957d402a5a5c92b45ff8e650077ebd01273",
"template_name":"DefaultVideoClassificationTemplate",
"template_task":"CLASSIFICATION",
"template_subtask":"MULTICLASS",
"problem_taskType":"classification",
"problem_taskSubType":"multiClass"
}
\ No newline at end of file
{
"id": "b7b40c43-3879-4f78-980c-12b824034cc4",
"schema": "https://metadata.datadrivendiscovery.org/schemas/v0/pipeline.json",
"created": "2019-05-17T20:20:42.321786Z",
"inputs": [
{
"name": "input dataset"
}
],
"outputs": [
{
"data": "steps.5.produce",
"name": "predictions of input dataset"
}
],
"steps": [
{
"type": "PRIMITIVE",
"primitive": {
"id": "4b42ce1e-9b98-4a25-b68e-fad13311eb65",
"version": "0.3.0",
"python_path": "d3m.primitives.data_transformation.dataset_to_dataframe.Common",
"name": "Extract a DataFrame from a Dataset",
"digest": "e557fc176af67913ed241dbe793cf986439157cfe9c4a12f1aafcba3c07b8b6f"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "inputs.0"
}
},
"outputs": [
{
"id": "produce"
}
]
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version": "0.2.0",
"python_path": "d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name": "Extracts columns by semantic type",
"digest": "d4fa27a3e215b89d2231d344ec0202e1b8e41ad322deb79c68fd6ca673eae3f7"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.0.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"semantic_types": {
"type": "VALUE",
"data": [
"https://metadata.datadrivendiscovery.org/types/PrimaryKey",
"https://metadata.datadrivendiscovery.org/types/FileName"
]
}
}
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version": "0.2.0",
"python_path": "d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name": "Extracts columns by semantic type",
"digest": "d4fa27a3e215b89d2231d344ec0202e1b8e41ad322deb79c68fd6ca673eae3f7"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.0.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"semantic_types": {
"type": "VALUE",
"data": [
"https://metadata.datadrivendiscovery.org/types/TrueTarget"
]
}
}
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "a29b0080-aeff-407d-9edb-0aa3eefbde01",
"version": "0.2.0",
"python_path": "d3m.primitives.data_preprocessing.video_reader.DataFrameCommon",
"name": "Columns video reader",
"digest": "5a2961b4974a1af0dd503f90259474c6e5ed44a9a61a44163de9c8560d606b63"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.1.produce"
}
},
"outputs": [
{
"id": "produce"
}
]
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "dsbox-featurizer-image-inceptionV3",
"version": "1.5.3",
"python_path": "d3m.primitives.feature_extraction.inceptionV3_image_feature.DSBOX",
"name": "DSBox Image Featurizer inceptionV3",
"digest": "11f7f28a3561816a357c5851c8ea560572271bfaddf33c018336bd1dc27af6e2"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.3.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"use_limitation": {
"type": "VALUE",
"data": false
}
}
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "dsbox-featurizer-video-classification-lstm",
"version": "1.5.3",
"python_path": "d3m.primitives.classification.lstm.DSBOX",
"name": "DSBox Video Classification LSTM",
"digest": "eb32157e614fd864d5b87da8417aecf079cf4691363aad37b4b55d3b64d758e0"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.4.produce"
},
"outputs": {
"type": "CONTAINER",
"data": "steps.2.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"LSTM_units": {
"type": "VALUE",
"data": 1024
},
"epochs": {
"type": "VALUE",
"data": 100
}
}
}
],
"name": "DefaultVideoClassificationTemplate:5019768584",
"description": "",
"digest": "6628738546bd0eee059852e600276926da67feebf73a6de89fa79f2ae831729a"
}
......@@ -23,7 +23,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"precondition": [],
......@@ -324,5 +324,5 @@
}
},
"structural_type": "dsbox.datapreprocessing.featurizer.image.video_classification.LSTM",
"digest": "f07059edb9e5f6ebb9d648d0c95be77b7233c16dc21fec45e28c4e4f94f029fc"
"digest": "41976ee5681a5fd27bd1a3b9c9f74ae305ad13a05f7826b29d75288fdaee94a9"
}
{
"problem": "DA_poverty_estimation_problem",
"full_inputs": ["DA_poverty_estimation_dataset"],
"train_inputs": ["DA_poverty_estimation_dataset_TRAIN"],
"test_inputs": ["DA_poverty_estimation_dataset_TEST"],
"score_inputs": ["DA_poverty_estimation_dataset_SCORE"]
}
\ No newline at end of file
......@@ -23,7 +23,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"schema": "https://metadata.datadrivendiscovery.org/schemas/v0/primitive.json",
......@@ -215,5 +215,5 @@
}
},
"structural_type": "dsbox.datapreprocessing.cleaner.wikifier.Wikifier",
"digest": "f8111370e158cfef3bc30f4862e9c8c1c897762b2b0e3e98faed6af7ad4e6a9b"
"digest": "6815d196d4b9a11bcb272924c9b92c23bf3ecf24a3df9adfa16273898a9d51e6"
}
......@@ -24,7 +24,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"location_uris": [],
......@@ -298,5 +298,5 @@
},
"structural_type": "dsbox.datapreprocessing.cleaner.cleaning_featurizer.CleaningFeaturizer",
"description": "A cleaning featurizer for imperfect data. Capabilities of this featurizer include:\n\n+ Split a column with compound string values (e.g. \"118,32\") to multiple columns.\n+ Split a date column into year, month, day, day-of-week.\n+ Split American phone number column into area code, prefix, and number.\n+ Split alphanumeric columns into multiple columns.\n\nThis primitive requires d3m.primitives.schema_discovery.profiler.DSBOX profiler.\n\nAttributes\n----------\nmetadata : PrimitiveMetadata\n Primitive's metadata. Available as a class attribute.\nlogger : Logger\n Primitive's logger. Available as a class attribute.\nhyperparams : Hyperparams\n Hyperparams passed to the constructor.\nrandom_seed : int\n Random seed passed to the constructor.\ndocker_containers : Dict[str, DockerContainer]\n A dict mapping Docker image keys from primitive's metadata to (named) tuples containing\n container's address under which the container is accessible by the primitive, and a\n dict mapping exposed ports to ports on that address.\nvolumes : Dict[str, str]\n A dict mapping volume keys from primitive's metadata to file and directory paths\n where downloaded and extracted files are available to the primitive.\ntemporary_directory : str\n An absolute path to a temporary directory a primitive can use to store any files\n for the duration of the current pipeline run phase. Directory is automatically\n cleaned up after the current pipeline run phase finishes.",
"digest": "38b0b6c1d871d749e29362f8658907fc7817f545435f5da3ace09365f8756707"
"digest": "ff8b0fe4e7bab01bab1bda1d93fa0575109ba3b183943cfdf54237888b0d8e5c"
}
......@@ -20,7 +20,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/[email protected]5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"location_uris": [],
......@@ -176,5 +176,5 @@
},
"structural_type": "dsbox.datapreprocessing.cleaner.column_fold.FoldColumns",
"description": "A column folding primitive for imperfect data. Fold multiple columns into one column based on common column name prefix.\nFor example, columns with names 'month-jan', 'month-feb', 'month-mar' and so on are folded into one column named 'month'.\"\n\nAttributes\n----------\nmetadata : PrimitiveMetadata\n Primitive's metadata. Available as a class attribute.\nlogger : Logger\n Primitive's logger. Available as a class attribute.\nhyperparams : Hyperparams\n Hyperparams passed to the constructor.\nrandom_seed : int\n Random seed passed to the constructor.\ndocker_containers : Dict[str, DockerContainer]\n A dict mapping Docker image keys from primitive's metadata to (named) tuples containing\n container's address under which the container is accessible by the primitive, and a\n dict mapping exposed ports to ports on that address.\nvolumes : Dict[str, str]\n A dict mapping volume keys from primitive's metadata to file and directory paths\n where downloaded and extracted files are available to the primitive.\ntemporary_directory : str\n An absolute path to a temporary directory a primitive can use to store any files\n for the duration of the current pipeline run phase. Directory is automatically\n cleaned up after the current pipeline run phase finishes.",
"digest": "b2a7f6b4604f276ed56fe1db821bfdc8bee6d2cad7d2f897bf7613d9d21d984e"
"digest": "944b5cdd8a3d662512a055f9c1cf1648bdc503e2b759e417ec1edef042fe5389"