demo.py 9.67 KB
Newer Older
Vincent Fazio's avatar
Vincent Fazio committed
1 2
#!/usr/bin/env python3
import sys
3 4 5 6
from types import SimpleNamespace
from nvcl_kit.reader import NVCLReader
from nvcl_kit.asud import get_asud_record
import yaml
Vincent Fazio's avatar
Vincent Fazio committed
7 8

#
9 10 11 12 13 14 15 16 17 18 19 20
# A very rough script to demonstrate 'nvcl_kit' 
#
# Linux command line instructions:
#
# git clone https://gitlab.com/csiro-geoanalytics/python-shared/nvcl_kit.git
# cd nvcl_kit
# mkdir venv
# python3 -m venv ./venv
# . ./venv/bin/activate
# pip install -U pip
# pip install -r requirements.txt
# ./demo.py
Vincent Fazio's avatar
Vincent Fazio committed
21
#
22
#
Vincent Fazio's avatar
Vincent Fazio committed
23

24 25 26 27 28 29 30
# Provider list. Format is (WFS service URL, NVCL service URL, bounding box coords, local filtering, WFS version, max boreholes))
prov_list = [ ("http://www.mrt.tas.gov.au:80/web-services/ows", "http://www.mrt.tas.gov.au/NVCLDataServices/", { "west": 143.75, "south": -43.75, "east": 148.75, "north": -39.75 }, False, "1.1.0", 20),
              ("http://geology.data.vic.gov.au/nvcl/ows", "http://geology.data.vic.gov.au/NVCLDataServices", None, False, "1.1.0", 20),
              ("https://gs.geoscience.nsw.gov.au/geoserver/ows", "https://nvcl.geoscience.nsw.gov.au/NVCLDataServices", None, False, "1.1.0", 20),
              ("https://geology.information.qld.gov.au/geoserver/ows", "https://geology.information.qld.gov.au/NVCLDataServices", None, False, "1.1.0", 20),
              ("http://geology.data.nt.gov.au:80/geoserver/ows", "http://geology.data.nt.gov.au:80/NVCLDataServices", None, True, "2.0.0", 20),
              ("https://sarigdata.pir.sa.gov.au/geoserver/ows", "https://sarigdata.pir.sa.gov.au/nvcl/NVCLDataServices",None, False, "1.1.0", 20),
31
              # NB: Western Australia's DMIRS only supports WFS v2.0.0
32 33
              ("http://geossdi.dmp.wa.gov.au/services/ows",  "http://geossdi.dmp.wa.gov.au/NVCLDataServices", None, False, "2.0.0", 20)
]
Vincent Fazio's avatar
Vincent Fazio committed
34 35


36
def do_demo(wfs, nvcl, bbox, local_filt, version, max):
37
    print("\n\n***", wfs, "***\n")
38 39

    # Assemble parameters
Vincent Fazio's avatar
Vincent Fazio committed
40
    param = SimpleNamespace()
41
    # NB: If you set USE_LOCAL_FILTERING to true then WFS_VERSION must be 2.0.0
42
    param.USE_LOCAL_FILTERING = local_filt
Vincent Fazio's avatar
Vincent Fazio committed
43
    param.WFS_URL = wfs
44
    param.WFS_VERSION = version
Vincent Fazio's avatar
Vincent Fazio committed
45 46 47
    param.NVCL_URL = nvcl
    if bbox:
        param.BBOX= bbox
48 49 50
    param.MAX_BOREHOLES = max

    # Initialise reader
Vincent Fazio's avatar
Vincent Fazio committed
51 52
    reader = NVCLReader(param)

53
    # Check for failure
Vincent Fazio's avatar
Vincent Fazio committed
54 55 56
    if not reader.wfs:
        print("ERROR!", wfs, nvcl)

57
    # Get boreholes list
Vincent Fazio's avatar
Vincent Fazio committed
58
    bh_list = reader.get_boreholes_list()
59
    print("len(bh_list) = ", len(bh_list))
60

61
    # Print borehole details and relevant records from Australian Stratigraphic Units Database (https://asud.ga.gov.au/)
62 63 64 65 66 67 68 69 70 71 72
    for bh in bh_list[:5]:
        print("\nBOREHOLE:")
        print(yaml.dump(bh))
        print("-"*80)
        a_rec = get_asud_record(bh['x'], bh['y'])
        print("\nAUSTRALIAN STRATIGRAPHIC UNITS DATABASE RECORD:")
        if a_rec is not None:
            print(yaml.dump(a_rec))
        else:
            print("Not found")
        print("="*80)
Vincent Fazio's avatar
Vincent Fazio committed
73 74 75

    # Get list of NVCL ids
    nvcl_id_list = reader.get_nvcl_id_list()
76 77 78 79
    print("len(nvcl_id_list) = ", len(nvcl_id_list))
    print("nvcl_id_list[:5] = ", nvcl_id_list[:5])

    # Exit if no nvcl ids found
Vincent Fazio's avatar
Vincent Fazio committed
80 81 82
    if not nvcl_id_list:
        print("!!!! No NVCL ids for", nvcl)
        return
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97

    # Some nvcl ids do not have any data - find the first which has data
    imagelog_data_list = []
    nvcl_id = ""
    for n_id in nvcl_id_list:
        imagelog_data_list = reader.get_imagelog_data(n_id)
        if imagelog_data_list:
            nvcl_id = n_id
            break

    # Exit if couldn't find valid data
    if not imagelog_data_list:
        print("!!!! No NVCL data for", nvcl)
        return

Vincent Fazio's avatar
Vincent Fazio committed
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
    for ild in imagelog_data_list[:10]:
        print(ild.log_id,
              ild.log_name,
              ild.log_type,
              ild.algorithmout_id)

    # Analysis class has 2 parts:
    # 1. Min1,2,3 = 1st, 2nd, 3rd most common mineral
    #    OR Grp1,2,3 = 1st, 2nd, 3rd most common group of minerals
    # 2. uTSAV = visible light, uTSAS = shortwave IR, uTSAT = thermal IR
    #
    # These combine to give us a class name such as 'Grp1 uTSAS'
    #
    # Here we extract data for log type '1' and 'Grp1 uTSAS'
    HEIGHT_RESOLUTION = 20.0
    ANALYSIS_CLASS = 'Grp1 uTSAS'
    LOG_TYPE = '1'
    for ild in imagelog_data_list[:10]:
        if ild.log_type == LOG_TYPE and ild.log_name == ANALYSIS_CLASS:
            print('get_borehole_data()')
118
            # Get top 5 minerals at each depth
119
            bh_data = reader.get_borehole_data(ild.log_id, HEIGHT_RESOLUTION, ANALYSIS_CLASS, top_n=5)
Vincent Fazio's avatar
Vincent Fazio committed
120
            for depth in bh_data:
121 122 123 124
                for meas in bh_data[depth]:
                    print("At {} metres: class={}, abundance={}, mineral={}, colour={}".format(depth, meas.className,
                      meas.classCount, meas.classText, meas.colour))
                print()
Vincent Fazio's avatar
Vincent Fazio committed
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259

    print('get_spectrallog_data()')
    spectrallog_data_list = reader.get_spectrallog_data(nvcl_id)
    for sld in spectrallog_data_list[:4]:
        print(sld.log_id,
              sld.log_name,
              sld.wavelength_units,
              sld.sample_count,
              sld.script,
              sld.script_raw,
              sld.wavelengths)

    print('get_profilometer_data()')
    profilometer_data_list = reader.get_profilometer_data(nvcl_id)
    for pdl in profilometer_data_list[:10]:
        print(pdl.log_id,
              pdl.log_name,
              pdl.max_val,
              pdl.min_val,
              pdl.floats_per_sample,
              pdl.sample_count)

    print('get_dataset_list()')
    dataset_list = reader.get_dataset_list(nvcl_id)
    for dataset in dataset_list[:10]:
        print(dataset.dataset_id,
              dataset.dataset_name,
              dataset.borehole_uri,
              dataset.tray_id,
              dataset.section_id,
              dataset.domain_id)

    print('get_datasetid_list()')
    datasetid_list = reader.get_datasetid_list(nvcl_id)
    for dataset_id in datasetid_list[:5]:
        print('dataset_id:', dataset_id)

        # GET_MOSAIC_IMGLOGS, GET_MOSAIC_IMAGE
        img_log_list = reader.get_mosaic_imglogs(dataset_id)
        print('get_mosaic_imglogs() ', img_log_list)
        for img_log in img_log_list[:10]:
            print(img_log.log_id,
                  img_log.log_name,
                  img_log.sample_count)
            html = reader.get_mosaic_image(img_log.log_id)
            print('get_mosaic_image()', html[:400])


        # GET_TRAY_THUMBNAIL_IMGLOGS, GET_TRAY_THUMB_HTML, GET_TRAY_THUMB_JPG
        # & GET_TRAY_DEPTHS
        print('get_tray_thumb_imglogs()')
        img_log_list = reader.get_tray_thumb_imglogs(dataset_id)
        for img_log in img_log_list[:10]:
            print(img_log.log_id,
                  img_log.log_name,
                  img_log.sample_count)
            html = reader.get_tray_thumb_html(dataset_id, img_log.log_id)
            print('get_tray_thumb_html()', html[:400])
            jpg = reader.get_tray_thumb_jpg(img_log.log_id)
            print('get_tray_thumb_jpg()', repr(jpg)[:100])
            depth_list = reader.get_tray_depths(img_log.log_id)
            print('get_tray_depths():')
            for depth in depth_list[:10]:
                print(depth.sample_no,
                      depth.start_value,
                      depth.end_value)


        # GET_TRAY_IMGLOGS, GET_TRAY_THUMB_HTML & GET_TRAY_DEPTHS
        print('get_tray_imglogs()')
        img_log_list = reader.get_tray_imglogs(dataset_id)
        for img_log in img_log_list[:10]:
            print(img_log.log_id,
                  img_log.log_name,
                  img_log.sample_count)
            html = reader.get_tray_thumb_html(dataset_id, img_log.log_id)
            print('get_tray_thumb_html()', html[:400])
            depth_list = reader.get_tray_depths(img_log.log_id)
            print('get_tray_depths():')
            for depth in depth_list[:10]:
                print(depth.sample_no,
                      depth.start_value,
                      depth.end_value)


        # GET_IMAGERY_IMGLOGS
        print('get_imagery_imglogs()')
        img_log_list = reader.get_imagery_imglogs(dataset_id)
        for img_log in img_log_list[:10]:
            print(img_log.log_id,
                  img_log.log_name,
                  img_log.sample_count)
            print('get_imagery_logs()', html[:400])


        # GET_SCALAR_LOGS & PLOT_SCALAR_PNG
        print('get_scalar_logs()')
        scalar_log_list = reader.get_scalar_logs(dataset_id)
        for scalar_log in scalar_log_list[:10]:
            print(scalar_log.log_id,
                  scalar_log.log_name)
            png = reader.plot_scalar_png(scalar_log.log_id)
            print('plot_scalar_png()', repr(png)[:100])


        # PLOT_SCALARS_HTML
        log_id_list = [scalar_log.log_id for scalar_log in scalar_log_list]
        html = reader.plot_scalars_html(log_id_list)
        print('plot_scalars_html()', html[:400])


        # GET_SCALAR_LOGS & GET_SCALAR_DATA
        sca_log_list = reader.get_scalar_logs(dataset_id)
        print('get_scalar_logs()', sca_log_list[:10])
        log_id_list = [sca_log.log_id for sca_log in sca_log_list][:4]
        csv = reader.get_scalar_data(log_id_list)
        print('get_scalar_data()', csv[:400])


        # GET_SAMPLED_SCALAR_DATA
        for sca_log in sca_log_list[:5]:
            sampled_data = reader.get_sampled_scalar_data(sca_log.log_id,
                                                     outputformat='json',
                                                     startdepth=0,
                                                     enddepth=2000,
                                                     interval=100)
            print('get_sampled_scalar_data()', sampled_data[:400])


#
# MAIN PART OF SCRIPT
#
if __name__ == "__main__":

    # Loop over all the providers
260 261
    for prov_info in prov_list:
        do_demo(*prov_info)
Vincent Fazio's avatar
Vincent Fazio committed
262