Merge branch 'phx-246-update-config-string' into 'master' (13c8a679) · Commits · SKAO / Science Data Processor / SDP Scripting Library

.make @ c38a4d0c

Original line number	Diff line number	Diff line
		Subproject commit 916024d9b7557ea52324a5c2c458dfc1d7a989eb
		Subproject commit c38a4d0c11b18a778c6d79ec26029fef22e85155

tests/data/assign_resources_config.json

0 → 100644

+223 −0

Original line number	Diff line number	Diff line
		{
		"interface":"https://schema.skao.int/ska-sdp-assignres/1.1",

		"resources": {
		"csp_links": [ 1, 2, 3, 4 ],
		"receptors": ["C10", "C136", "C1", "C217", "C13", "C42"],
		"receive_nodes": 1
		},

		"execution_block": {
		"eb_id": "eb-mvp01-20200325-00001",
		"max_length": 100.0,
		"context": {
		},
		"beams": [
		{
		"beam_id": "vis0",
		"function": "visibilities"
		},
		{
		"beam_id": "pss1",
		"search_beam_id": 1,
		"function": "pulsar search"
		},
		{
		"beam_id": "pss2",
		"search_beam_id": 2,
		"function": "pulsar search"
		},
		{
		"beam_id": "pst1",
		"timing_beam_id": 1,
		"function": "pulsar timing"
		},
		{
		"beam_id": "pst2",
		"timing_beam_id": 2,
		"function": "pulsar timing"
		},
		{
		"beam_id": "vlbi1",
		"vlbi_beam_id": 1,
		"function": "vlbi"
		}
		],
		"scan_types": [
		{
		"scan_type_id": ".default",
		"beams": {
		"vis0": {
		"channels_id": "vis_channels",
		"polarisations_id": "all",
		"visibility_beam_id": 1
		},
		"pss1": {
		"field_id": "pss_field_0",
		"channels_id": "pulsar_channels",
		"polarisations_id": "all"
		},
		"pss2": {
		"field_id": "pss_field_1",
		"channels_id": "pulsar_channels",
		"polarisations_id": "all"
		},
		"pst1": {
		"field_id": "pst_field_0",
		"channels_id": "pulsar_channels",
		"polarisations_id": "all"
		},
		"pst2": {
		"field_id": "pst_field_1",
		"channels_id": "pulsar_channels",
		"polarisations_id": "all"
		},
		"vlbi1": {
		"field_id": "vlbi_field",
		"channels_id": "vlbi_channels",
		"polarisations_id": "all"
		}
		}
		},
		{
		"scan_type_id": "target:a",
		"derive_from": ".default",
		"beams": {
		"vis0": {
		"field_id": "field_a"
		},
		"pss1": {}
		}
		}
		],
		"channels": [
		{
		"channels_id": "vis_channels",
		"spectral_windows": [
		{
		"spectral_window_id": "fsp_1_channels",
		"count": 4,
		"start": 0,
		"stride": 2,
		"freq_min": 350000000.0,
		"freq_max": 368000000.0,
		"link_map": [
		[
		0,
		0
		],
		[
		200,
		1
		],
		[
		744,
		2
		],
		[
		944,
		3
		]
		]
		}
		]
		},
		{
		"channels_id": "pulsar_channels",
		"spectral_windows": [
		{
		"spectral_window_id": "pulsar_fsp_channels",
		"count": 2,
		"start": 0,
		"stride": 2,
		"freq_min": 350000000.0,
		"freq_max": 368000000.0
		}
		]
		},
		{
		"channels_id": "vlbi_channels",
		"spectral_windows": [
		{
		"spectral_window_id": "vlbi_fsp_channels",
		"count": 2,
		"start": 0,
		"stride": 2,
		"freq_min": 350000000.0,
		"freq_max": 368000000.0
		}
		]
		}
		],
		"polarisations": [
		{
		"polarisations_id": "all",
		"corr_type": [
		"XX",
		"XY",
		"YY",
		"YX"
		]
		}
		],
		"fields": [
		{
		"field_id": "field_a",
		"phase_dir": {
		"target_name": "target_a",
		"reference_frame": "icrs",
		"attrs": {
		"c1": 123.0,
		"c2": -60.0,
		"epoch": 2000.0
		}
		},
		"pointing_fqdn": "low-tmc/telstate/0/pointing"
		}
		]
		},
		"processing_blocks":[
		{
		"pb_id":"pb-test-20200425-00000",
		"script":{
		"kind":"realtime",
		"name":"test-receive-addresses",
		"version":"0.3.3"
		},
		"parameters":{
		"length":10,
		"duration":20
		}
		},
		{
		"pb_id":"pb-test-20200425-00001",
		"script":{
		"kind":"realtime",
		"name":"test-new-receive",
		"version":"0.1.0"
		},
		"parameters":{
		"length":10,
		"duration":20
		},
		"dependencies": [
		{"pb_id": "pb-test-20200425-00000", "kind": ["calibration"]}
		]
		},
		{
		"pb_id":"pb-test-20200425-00002",
		"script":{
		"kind":"batch",
		"name":"test-dask",
		"version":"0.2.1"
		},
		"parameters":{
		"n_workers": 3
		},
		"dependencies": [
		{"pb_id": "pb-test-20200425-00000", "kind": ["calibration"]},
		{"pb_id": "pb-test-20200425-00001", "kind": ["science"]}
		]
		}
		]
		}
		No newline at end of file

tests/data/configuration_string.json

deleted100644 → 0

+0 −216

Original line number	Diff line number	Diff line
		{
		"eb_id":"eb-mvp01-20200325-00001",
		"max_length":100.0,
		"context":{

		},
		"beams":[
		{
		"beam_id":"vis0",
		"function":"visibilities"
		},
		{
		"beam_id":"pss1",
		"search_beam_id":1,
		"function":"pulsar search"
		},
		{
		"beam_id":"pss2",
		"search_beam_id":2,
		"function":"pulsar search"
		},
		{
		"beam_id":"pst1",
		"timing_beam_id":1,
		"function":"pulsar timing"
		},
		{
		"beam_id":"pst2",
		"timing_beam_id":2,
		"function":"pulsar timing"
		},
		{
		"beam_id":"vlbi1",
		"vlbi_beam_id":1,
		"function":"vlbi"
		}
		],
		"scan_types":[
		{
		"scan_type_id":".default",
		"beams":{
		"vis0":{
		"channels_id":"vis_channels",
		"polarisations_id":"all"
		},
		"pss1":{
		"field_id":"pss_field_0",
		"channels_id":"pulsar_channels",
		"polarisations_id":"all"
		},
		"pss2":{
		"field_id":"pss_field_1",
		"channels_id":"pulsar_channels",
		"polarisations_id":"all"
		},
		"pst1":{
		"field_id":"pst_field_0",
		"channels_id":"pulsar_channels",
		"polarisations_id":"all"
		},
		"pst2":{
		"field_id":"pst_field_1",
		"channels_id":"pulsar_channels",
		"polarisations_id":"all"
		},
		"vlbi1":{
		"field_id":"vlbi_field",
		"channels_id":"vlbi_channels",
		"polarisations_id":"all"
		}
		}
		},
		{
		"scan_type_id":"target:a",
		"derive_from":".default",
		"beams":{
		"vis0":{
		"field_id":"field_a"
		},
		"pss1": {}
		}
		}
		],
		"channels":[
		{
		"channels_id":"vis_channels",
		"spectral_windows":[
		{
		"spectral_window_id":"fsp_1_channels",
		"count":4,
		"start":0,
		"stride":2,
		"freq_min":350000000.0,
		"freq_max":368000000.0,
		"link_map":[
		[
		0,
		0
		],
		[
		200,
		1
		],
		[
		744,
		2
		],
		[
		944,
		3
		]
		]
		}
		]
		},
		{
		"channels_id":"pulsar_channels",
		"spectral_windows": [
		{
		"spectral_window_id":"pulsar_fsp_channels",
		"count":2,
		"start":0,
		"stride":2,
		"freq_min":350000000.0,
		"freq_max":368000000.0
		}
		]
		},
		{
		"channels_id":"vlbi_channels",
		"spectral_windows": [
		{
		"spectral_window_id":"vlbi_fsp_channels",
		"count":2,
		"start":0,
		"stride":2,
		"freq_min":350000000.0,
		"freq_max":368000000.0
		}
		]
		}
		],
		"polarisations":[
		{
		"polarisations_id":"all",
		"corr_type":[
		"XX",
		"XY",
		"YY",
		"YX"
		]
		}
		],
		"fields":[
		{
		"field_id":"field_a",
		"phase_dir":{
		"ra":[
		123,
		0.1
		],
		"dec":[
		-60,
		0.1
		],
		"reference_time":"...",
		"reference_frame":"ICRF3"
		},
		"pointing_fqdn":"low-tmc/telstate/0/pointing"
		}
		],
		"processing_blocks":[
		{
		"pb_id":"pb-test-20200425-00000",
		"script":{
		"kind":"realtime",
		"name":"test-receive-addresses",
		"version":"0.3.3"
		},
		"parameters":{
		"length":10,
		"duration":20
		}
		},
		{
		"pb_id":"pb-test-20200425-00001",
		"script":{
		"kind":"realtime",
		"name":"test-new-receive",
		"version":"0.1.0"
		},
		"parameters":{
		"length":10,
		"duration":20
		},
		"dependencies": [
		{"pb_id": "pb-test-20200425-00000", "kind": ["calibration"]}
		]
		},
		{
		"pb_id":"pb-test-20200425-00002",
		"script":{
		"kind":"batch",
		"name":"test-dask",
		"version":"0.2.1"
		},
		"parameters":{
		"n_workers": 3
		},
		"dependencies": [
		{"pb_id": "pb-test-20200425-00000", "kind": ["calibration"]},
		{"pb_id": "pb-test-20200425-00001", "kind": ["science"]}
		]
		}
		]
		}
		No newline at end of file

tests/test_processing_block.py

+6 −5

Original line number	Diff line number	Diff line
		@@ -42,7 +42,8 @@ def test_get_scan_types(pb):
		Test that scan_types are correctly updated using
		default_scan_types and channel information
		"""
		config = read_configuration_string(inconsistent_beam=True)
		ar_string = read_configuration_string(inconsistent_beam=True)
		config = ar_string["execution_block"]
		channels = config.get("channels")

		# check that the scan_type which derives from a pseudo (default) scan type
		@@ -75,7 +76,7 @@ def test_get_scan_types(pb):

		def test_get_dependency(config_db_client):
		"""
		Return the expected dependencies defined in configuration_string.json
		Return the expected dependencies defined in assign_resources_config.json
		"""
		with patch(
		"ska_sdp_scripting.processing_block.new_config_client"
		@@ -112,7 +113,7 @@ def test_pb_parameter_validation(config_db_client):

		By default, we create a processing block of test_dask,
		which has the id of "pb-test-20200425-00002" in
		tests/data/configuration_string.json
		tests/data/assign_resources_config.json
		"""
		with patch(
		"ska_sdp_scripting.processing_block.new_config_client"
		@@ -136,7 +137,7 @@ def test_pb_parameter_validation_bad_arg_type(wrong_type, config_db_client):

		By default, we create a processing block of test_dask,
		which has the id of "pb-test-20200425-00002" in
		tests/data/configuration_string.json
		tests/data/assign_resources_config.json
		"""
		with patch(
		"ska_sdp_scripting.processing_block.new_config_client"
		@@ -163,7 +164,7 @@ def test_pb_parameter_validation_bad_key(config_db_client):

		By default, we create a processing block of test_dask,
		which has the id of "pb-test-20200425-00002" in
		tests/data/configuration_string.json
		tests/data/assign_resources_config.json
		"""
		bad_keys = ["some_key", "some_other_key"]
		with patch(

tests/test_receive_addresses.py

+4 −3

Original line number	Diff line number	Diff line
		@@ -98,7 +98,7 @@ SCAN_TYPES = [
		]

		# scan types returned by pb.get_scan_types() when
		# configuration_string.json file is used
		# assign_resources_config.json file is used
		EXPECTED_SCAN_TYPES = [
		{
		"beams": {
		@@ -117,6 +117,7 @@ EXPECTED_SCAN_TYPES = [
		"stride": 2,
		}
		],
		"visibility_beam_id": 1,
		},
		"pss1": {
		"channels_id": "pulsar_channels",
		@@ -240,7 +241,7 @@ RECV_ADDRESS = {
		}

		# These are the receive addresses generated when
		# the configuration_string.json file is used for scan_types
		# the assign_resources_config.json file is used for scan_types
		RECV_ADDR_FROM_CONFIG_STRING = {
		"target:a": {
		"vis0": {
		@@ -318,7 +319,7 @@ RECV_ADDR_FROM_CONFIG_STRING = {
		}

		# These are the receive addresses generated when
		# the configuration_string.json file is used for scan_types
		# the assign_resources_config.json file is used for scan_types
		# and inconsistent_beam is set to True for create_eb_pb()
		RECV_ADDR_DUPLICATE_BEAM = {
		"target:a": RECV_ADDR_FROM_CONFIG_STRING["target:a"],