AVRO Schema Editor.ipynb 13.1 KB
Newer Older
Marc Matt's avatar
Marc Matt committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AVRO Schema Editor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What is an AVRO schema?\n",
    "AVRO is a serialized Java object to store data in a structured way. More information can be found here:\n",
    "http://avro.apache.org/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to edit a schema\n",
    "* files can be loaded using magic quotes in this notebook\n",
    "  * %%writefile path_to_file writes content in cell to path_to_file\n",
    "  * %load path_to_file loads file from directory into cell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define paths to avro files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_defined = './schemas/defined'\n",
    "path_to_data_types = './schemas/data_types'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### list all avro schemas already available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['person.avsc',\n 'licence.avsc',\n 'licence_area.avsc',\n 'farming_area.avsc',\n 'shop.avsc',\n 'address.avsc',\n 'user.avsc']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import simplejson\n",
    "os.listdir(path_to_defined)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### define filename to add / change\n",
    "if you set a filename here, that does not exist, a new schema will be created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_to_change = 'test.avsc'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### change cell headers\n",
    "the cell headers need to be changed with every execution. Replace the existing two rows with the following row:\n",
    "```\n",
    "%load $path_to_defined/$file_to_change\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "'./schemas/defined/test.avsc' was not found in history, as a file, url, nor in the user namespace.",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mSyntaxError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m~/anaconda3/envs/avro-generator/lib/python3.6/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mfind_user_code\u001b[0;34m(self, target, raw, py_only, skip_encoding_cookie, search_ns)\u001b[0m\n\u001b[1;32m   3280\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m                                              \u001b[0;31m# User namespace\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3281\u001b[0;31m             \u001b[0mcodeobj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muser_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3282\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mSyntaxError\u001b[0m: invalid syntax (<string>, line 1)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-5d16b7a1abde>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'load'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'$path_to_defined/$file_to_change'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m {\n\u001b[1;32m      3\u001b[0m     \"fields\": [\n\u001b[1;32m      4\u001b[0m         {\n\u001b[1;32m      5\u001b[0m             \u001b[0;34m\"doc\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"unix timestamp of message generation (timezone UTC)\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/avro-generator/lib/python3.6/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line, _stack_depth)\u001b[0m\n\u001b[1;32m   2129\u001b[0m                 \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2130\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2131\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2132\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2133\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<decorator-gen-47>\u001b[0m in \u001b[0;36mload\u001b[0;34m(self, arg_s)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/avro-generator/lib/python3.6/site-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    185\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 187\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    188\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    189\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/avro-generator/lib/python3.6/site-packages/IPython/core/magics/code.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0msearch_ns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'n'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mopts\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 335\u001b[0;31m         \u001b[0mcontents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_user_code\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msearch_ns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msearch_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    336\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    337\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;34m's'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mopts\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/avro-generator/lib/python3.6/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mfind_user_code\u001b[0;34m(self, target, raw, py_only, skip_encoding_cookie, search_ns)\u001b[0m\n\u001b[1;32m   3282\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3283\u001b[0m             raise ValueError((\"'%s' was not found in history, as a file, url, \"\n\u001b[0;32m-> 3284\u001b[0;31m                                 \"nor in the user namespace.\") % target)\n\u001b[0m\u001b[1;32m   3285\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3286\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcodeobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: './schemas/defined/test.avsc' was not found in history, as a file, url, nor in the user namespace."
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "%load $path_to_defined/$file_to_change\n",
    "{\n",
    "    \"fields\": [\n",
    "        {\n",
    "            \"doc\": \"unix timestamp of message generation (timezone UTC)\",\n",
    "            \"name\": \"meta_extracted_at_time_utc\",\n",
    "            \"type\": \"long\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"meta_transaction_action\",\n",
    "            \"type\": \"de.env.data.types.meta_transaction_action\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"string identifying system that pushed the message (e.g. pusher_1, pusher_2, etc.) to make possible debugging in case of error easier.\",\n",
    "            \"name\": \"meta_source_system\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"unique identifier of address object\",\n",
    "            \"name\": \"address_id\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"name of street for address object\",\n",
    "            \"name\": \"street\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"house number of address object\",\n",
    "            \"name\": \"street_no\",\n",
    "            \"type\": [\n",
    "                \"null\",\n",
    "                \"string\"\n",
    "            ]\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"zip code of address object\",\n",
    "            \"name\": \"zip_code\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"city of address object\",\n",
    "            \"name\": \"city\",\n",
    "            \"type\": \"string\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"iso3 code of country of address object\",\n",
    "            \"name\": \"country_iso3\",\n",
    "            \"type\": \"de.env.data.types.fixed_iso3\"\n",
    "        },\n",
    "        {\n",
    "            \"doc\": \"map of regions of address object\",\n",
    "            \"name\": \"regions\",\n",
    "            \"type\": [\n",
    "                \"null\",\n",
    "                {\n",
    "                    \"type\": \"map\",\n",
    "                    \"values\": \"string\"\n",
    "                }\n",
    "            ]\n",
    "        }\n",
    "    ],\n",
    "    \"name\": \"Address\",\n",
    "    \"namespace\": \"de.env.data\",\n",
    "    \"type\": \"record\"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save changed / created schema to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "schema = _\n",
    "dumps = json.dumps(schema)\n",
    "\n",
    "datafile = open(path_to_defined + \"/\" + file_to_change, \"w\")\n",
    "# magic happens here to make it pretty-printed\n",
    "datafile.write(simplejson.dumps(simplejson.loads(dumps), indent=4, sort_keys=True))\n",
    "datafile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}