Commit 6dd1f49b authored by Sujen's avatar Sujen

Merge branch 'isi_v2019.6.7' into 'master'

Isi v2019.6.7 primitive and sample pipelines final update

See merge request datadrivendiscovery/primitives!263
parents 91036888 35c21f4a
......@@ -22,7 +22,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"schema": "https://metadata.datadrivendiscovery.org/schemas/v0/primitive.json",
......@@ -185,5 +185,5 @@
}
},
"structural_type": "dsbox.datapostprocessing.ensemble_voting.EnsembleVoting",
"digest": "2cb4d81bfe64821ca655d329cc8fff7e8054764eecf5979ac7e34e689ba778c3"
"digest": "f9ec132819a2daebddc99a2fecac011a89b0e88c14ea7173801b797bd9efca49"
}
{
"id":"6d5af388-e635-483f-9427-515e5e9e7476",
"schema":"https://metadata.datadrivendiscovery.org/schemas/v0/pipeline.json",
"created":"2019-06-10T22:36:16.413877Z",
"inputs":[
{
"name":"input dataset"
}
],
"outputs":[
{
"data":"steps.6.produce",
"name":"predictions of input dataset"
}
],
"steps":[
{
"type":"PRIMITIVE",
"primitive":{
"id":"f31f8c1f-d1c5-43e5-a4b2-2ae4a761ef2e",
"version":"0.2.0",
"python_path":"d3m.primitives.data_transformation.denormalize.Common",
"name":"Denormalize datasets",
"digest":"948abcebe21aa6765ecc8caf73c314058a623be2665cf69eb57c7d7662db23c7"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"inputs.0"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"starting_resource":{
"type":"VALUE",
"data":null
},
"recursive":{
"type":"VALUE",
"data":true
},
"many_to_many":{
"type":"VALUE",
"data":false
},
"discard_not_joined_tabular_resources":{
"type":"VALUE",
"data":false
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"4b42ce1e-9b98-4a25-b68e-fad13311eb65",
"version":"0.3.0",
"python_path":"d3m.primitives.data_transformation.dataset_to_dataframe.Common",
"name":"Extract a DataFrame from a Dataset",
"digest":"9935952c663b051a9a5bf25045adb3ce7803b724f8397306dabd458a3350627d"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.0.produce"
}
},
"outputs":[
{
"id":"produce"
}
]
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version":"0.2.0",
"python_path":"d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name":"Extracts columns by semantic type",
"digest":"9111567d84de0ffe913eee93395a4a3f1eceb2d8db600b39523257297f09a181"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.1.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"semantic_types":{
"type":"VALUE",
"data":[
"https://metadata.datadrivendiscovery.org/types/PrimaryKey",
"https://metadata.datadrivendiscovery.org/types/FileName"
]
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version":"0.2.0",
"python_path":"d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name":"Extracts columns by semantic type",
"digest":"9111567d84de0ffe913eee93395a4a3f1eceb2d8db600b39523257297f09a181"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.1.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"semantic_types":{
"type":"VALUE",
"data":[
"https://metadata.datadrivendiscovery.org/types/TrueTarget"
]
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"a29b0080-aeff-407d-9edb-0aa3eefbde01",
"version":"0.2.0",
"python_path":"d3m.primitives.data_preprocessing.video_reader.DataFrameCommon",
"name":"Columns video reader",
"digest":"67a102c1fb435cab3b2ddbe3e568d3835c09b01cee877791a123ce105dcb8c4f"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.2.produce"
}
},
"outputs":[
{
"id":"produce"
}
]
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"dsbox-featurizer-image-inceptionV3",
"version":"1.5.2",
"python_path":"d3m.primitives.feature_extraction.inceptionV3_image_feature.DSBOX",
"name":"DSBox Image Featurizer inceptionV3",
"digest":"6622a86ce5c6462cd7a6a7b258e369e22bf692754f7b1dc16a24d1aa4758af1a"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.4.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"use_limitation":{
"type":"VALUE",
"data":false
}
}
},
{
"type":"PRIMITIVE",
"primitive":{
"id":"dsbox-featurizer-video-classification-lstm",
"version":"1.5.2",
"python_path":"d3m.primitives.classification.lstm.DSBOX",
"name":"DSBox Video Classification LSTM",
"digest":"a275f2d85b93df3a1b85339ca6fe2916e346724af3dac4ad1ad57a53f823367a"
},
"arguments":{
"inputs":{
"type":"CONTAINER",
"data":"steps.5.produce"
},
"outputs":{
"type":"CONTAINER",
"data":"steps.3.produce"
}
},
"outputs":[
{
"id":"produce"
}
],
"hyperparams":{
"LSTM_units":{
"type":"VALUE",
"data":1024
},
"epochs":{
"type":"VALUE",
"data":100
}
}
}
],
"name":"DefaultVideoClassificationTemplate:5273959808",
"description":"",
"digest":"dc104869b2f7d2b489e2d088640b7957d402a5a5c92b45ff8e650077ebd01273",
"template_name":"DefaultVideoClassificationTemplate",
"template_task":"CLASSIFICATION",
"template_subtask":"MULTICLASS",
"problem_taskType":"classification",
"problem_taskSubType":"multiClass"
}
\ No newline at end of file
{
"id": "b7b40c43-3879-4f78-980c-12b824034cc4",
"schema": "https://metadata.datadrivendiscovery.org/schemas/v0/pipeline.json",
"created": "2019-05-17T20:20:42.321786Z",
"inputs": [
{
"name": "input dataset"
}
],
"outputs": [
{
"data": "steps.5.produce",
"name": "predictions of input dataset"
}
],
"steps": [
{
"type": "PRIMITIVE",
"primitive": {
"id": "4b42ce1e-9b98-4a25-b68e-fad13311eb65",
"version": "0.3.0",
"python_path": "d3m.primitives.data_transformation.dataset_to_dataframe.Common",
"name": "Extract a DataFrame from a Dataset",
"digest": "e557fc176af67913ed241dbe793cf986439157cfe9c4a12f1aafcba3c07b8b6f"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "inputs.0"
}
},
"outputs": [
{
"id": "produce"
}
]
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version": "0.2.0",
"python_path": "d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name": "Extracts columns by semantic type",
"digest": "d4fa27a3e215b89d2231d344ec0202e1b8e41ad322deb79c68fd6ca673eae3f7"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.0.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"semantic_types": {
"type": "VALUE",
"data": [
"https://metadata.datadrivendiscovery.org/types/PrimaryKey",
"https://metadata.datadrivendiscovery.org/types/FileName"
]
}
}
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "4503a4c6-42f7-45a1-a1d4-ed69699cf5e1",
"version": "0.2.0",
"python_path": "d3m.primitives.data_transformation.extract_columns_by_semantic_types.DataFrameCommon",
"name": "Extracts columns by semantic type",
"digest": "d4fa27a3e215b89d2231d344ec0202e1b8e41ad322deb79c68fd6ca673eae3f7"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.0.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"semantic_types": {
"type": "VALUE",
"data": [
"https://metadata.datadrivendiscovery.org/types/TrueTarget"
]
}
}
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "a29b0080-aeff-407d-9edb-0aa3eefbde01",
"version": "0.2.0",
"python_path": "d3m.primitives.data_preprocessing.video_reader.DataFrameCommon",
"name": "Columns video reader",
"digest": "5a2961b4974a1af0dd503f90259474c6e5ed44a9a61a44163de9c8560d606b63"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.1.produce"
}
},
"outputs": [
{
"id": "produce"
}
]
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "dsbox-featurizer-image-inceptionV3",
"version": "1.5.3",
"python_path": "d3m.primitives.feature_extraction.inceptionV3_image_feature.DSBOX",
"name": "DSBox Image Featurizer inceptionV3",
"digest": "11f7f28a3561816a357c5851c8ea560572271bfaddf33c018336bd1dc27af6e2"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.3.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"use_limitation": {
"type": "VALUE",
"data": false
}
}
},
{
"type": "PRIMITIVE",
"primitive": {
"id": "dsbox-featurizer-video-classification-lstm",
"version": "1.5.3",
"python_path": "d3m.primitives.classification.lstm.DSBOX",
"name": "DSBox Video Classification LSTM",
"digest": "eb32157e614fd864d5b87da8417aecf079cf4691363aad37b4b55d3b64d758e0"
},
"arguments": {
"inputs": {
"type": "CONTAINER",
"data": "steps.4.produce"
},
"outputs": {
"type": "CONTAINER",
"data": "steps.2.produce"
}
},
"outputs": [
{
"id": "produce"
}
],
"hyperparams": {
"LSTM_units": {
"type": "VALUE",
"data": 1024
},
"epochs": {
"type": "VALUE",
"data": 100
}
}
}
],
"name": "DefaultVideoClassificationTemplate:5019768584",
"description": "",
"digest": "6628738546bd0eee059852e600276926da67feebf73a6de89fa79f2ae831729a"
}
......@@ -23,7 +23,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"precondition": [],
......@@ -324,5 +324,5 @@
}
},
"structural_type": "dsbox.datapreprocessing.featurizer.image.video_classification.LSTM",
"digest": "f07059edb9e5f6ebb9d648d0c95be77b7233c16dc21fec45e28c4e4f94f029fc"
"digest": "41976ee5681a5fd27bd1a3b9c9f74ae305ad13a05f7826b29d75288fdaee94a9"
}
{
"problem": "DA_poverty_estimation_problem",
"full_inputs": ["DA_poverty_estimation_dataset"],
"train_inputs": ["DA_poverty_estimation_dataset_TRAIN"],
"test_inputs": ["DA_poverty_estimation_dataset_TEST"],
"score_inputs": ["DA_poverty_estimation_dataset_SCORE"]
}
\ No newline at end of file
......@@ -23,7 +23,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"schema": "https://metadata.datadrivendiscovery.org/schemas/v0/primitive.json",
......@@ -215,5 +215,5 @@
}
},
"structural_type": "dsbox.datapreprocessing.cleaner.wikifier.Wikifier",
"digest": "f8111370e158cfef3bc30f4862e9c8c1c897762b2b0e3e98faed6af7ad4e6a9b"
"digest": "6815d196d4b9a11bcb272924c9b92c23bf3ecf24a3df9adfa16273898a9d51e6"
}
......@@ -24,7 +24,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"location_uris": [],
......@@ -298,5 +298,5 @@
},
"structural_type": "dsbox.datapreprocessing.cleaner.cleaning_featurizer.CleaningFeaturizer",
"description": "A cleaning featurizer for imperfect data. Capabilities of this featurizer include:\n\n+ Split a column with compound string values (e.g. \"118,32\") to multiple columns.\n+ Split a date column into year, month, day, day-of-week.\n+ Split American phone number column into area code, prefix, and number.\n+ Split alphanumeric columns into multiple columns.\n\nThis primitive requires d3m.primitives.schema_discovery.profiler.DSBOX profiler.\n\nAttributes\n----------\nmetadata : PrimitiveMetadata\n Primitive's metadata. Available as a class attribute.\nlogger : Logger\n Primitive's logger. Available as a class attribute.\nhyperparams : Hyperparams\n Hyperparams passed to the constructor.\nrandom_seed : int\n Random seed passed to the constructor.\ndocker_containers : Dict[str, DockerContainer]\n A dict mapping Docker image keys from primitive's metadata to (named) tuples containing\n container's address under which the container is accessible by the primitive, and a\n dict mapping exposed ports to ports on that address.\nvolumes : Dict[str, str]\n A dict mapping volume keys from primitive's metadata to file and directory paths\n where downloaded and extracted files are available to the primitive.\ntemporary_directory : str\n An absolute path to a temporary directory a primitive can use to store any files\n for the duration of the current pipeline run phase. Directory is automatically\n cleaned up after the current pipeline run phase finishes.",
"digest": "38b0b6c1d871d749e29362f8658907fc7817f545435f5da3ace09365f8756707"
"digest": "ff8b0fe4e7bab01bab1bda1d93fa0575109ba3b183943cfdf54237888b0d8e5c"
}
......@@ -20,7 +20,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"location_uris": [],
......@@ -176,5 +176,5 @@
},
"structural_type": "dsbox.datapreprocessing.cleaner.column_fold.FoldColumns",
"description": "A column folding primitive for imperfect data. Fold multiple columns into one column based on common column name prefix.\nFor example, columns with names 'month-jan', 'month-feb', 'month-mar' and so on are folded into one column named 'month'.\"\n\nAttributes\n----------\nmetadata : PrimitiveMetadata\n Primitive's metadata. Available as a class attribute.\nlogger : Logger\n Primitive's logger. Available as a class attribute.\nhyperparams : Hyperparams\n Hyperparams passed to the constructor.\nrandom_seed : int\n Random seed passed to the constructor.\ndocker_containers : Dict[str, DockerContainer]\n A dict mapping Docker image keys from primitive's metadata to (named) tuples containing\n container's address under which the container is accessible by the primitive, and a\n dict mapping exposed ports to ports on that address.\nvolumes : Dict[str, str]\n A dict mapping volume keys from primitive's metadata to file and directory paths\n where downloaded and extracted files are available to the primitive.\ntemporary_directory : str\n An absolute path to a temporary directory a primitive can use to store any files\n for the duration of the current pipeline run phase. Directory is automatically\n cleaned up after the current pipeline run phase finishes.",
"digest": "b2a7f6b4604f276ed56fe1db821bfdc8bee6d2cad7d2f897bf7613d9d21d984e"
"digest": "944b5cdd8a3d662512a055f9c1cf1648bdc503e2b759e417ec1edef042fe5389"
}
......@@ -22,7 +22,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"precondition": [
......@@ -248,5 +248,5 @@
}
},
"structural_type": "dsbox.datapreprocessing.cleaner.labler.Labler",
"digest": "8454588a6701278903c26bdf5ad10d73957c95821d83b2cdf49238fb6694cddf"
"digest": "9c8ee8ef221782d6aa9344c042dbdbeb9b196eb22ff929c7e0fee6a6f47e9cbe"
}
......@@ -22,7 +22,7 @@
"installation": [
{
"type": "PIP",
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@af995be72ba1230788a04a27b3f31b9ec353d1d2#egg=dsbox-primitives"
"package_uri": "git+https://github.com/usc-isi-i2/dsbox-primitives@5da8179cd96d3b0a64c970f1ee43f2960667fa8b#egg=dsbox-primitives"
}
],
"precondition": [],
......@@ -247,5 +247,5 @@
}
},
"structural_type": "dsbox.datapreprocessing.featurizer.image.dataframe_to_tensor.DataFrameToTensor",
"digest": "c0c38adf82b5c0ca136758760f6bb81834d7dca61e6be5e64537552360a33f15"
"digest": "826d55b95c3b990c2e54b8489d99ed7810224868103edb276455860fe7342896"
}