diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da4e2db0f4b7a2c33ee7f863de770654022b2fe8..44c93d0405423404c7a2411795eed204a1fd8151 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,9 @@ python-lint: - when: always python-test: - image: registry.gitlab.com/ska-telescope/external/rascil/rascil-full:latest + image: + name: registry.gitlab.com/ska-telescope/external/rascil-main/rascil-full:latest + pull_policy: always before_script: - apt-get update && apt-get -y install cmake - pip3 install --upgrade pip && pip install -r requirements.txt -r requirements-test.txt @@ -39,11 +41,15 @@ python-test: # need to overwrite the following to to use the # local make docs targets docs-build: + before_script: + - pip3 install --upgrade pip && pip install -r requirements-test.txt -r ./docs/requirements.txt script: - make set-up-docs - make docs docs-pages: + before_script: + - pip3 install --upgrade pip && pip install -r requirements-test.txt -r ./docs/requirements.txt script: - make set-up-docs - make docs diff --git a/continuum_imaging/sp_1331/image_low.sh b/continuum_imaging/sp_1331/image_low.sh index 636c39c96570688f1a97c602c3c46d762f01031a..f5ce9a5f3341b073d1e3e479e5558524954c5d6e 100644 --- a/continuum_imaging/sp_1331/image_low.sh +++ b/continuum_imaging/sp_1331/image_low.sh @@ -8,6 +8,6 @@ python3 ${RASCIL}/rascil/apps/rascil_imager.py --clean_nmoment 3 --clean_facets --use_dask True --imaging_context ng --imaging_npixel 16384 --imaging_pol stokesI \ --clean_restored_output taylor \ --imaging_cellsize 1e-5 --imaging_weighting uniform --imaging_nchan 1 \ - --ingest_vis_nchan 100 --ingest_chan_per_blockvis 6 \ + --ingest_vis_nchan 100 --ingest_chan_per_vis 6 \ --ingest_msname ${results_dir}/scalar_gaussian_beams.ms \ --performance_file ${results_dir}/performance_rascil_imager.json diff --git a/requirements-test.txt b/requirements-test.txt index d03a9b498d2406d8cffbe7a44c48f3bc935ce34d..348d1acf7dc63b30cc7d6cb15f58f517abb89653 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,5 @@ pylint-junit pytest pytest-cov black -isort \ No newline at end of file +isort +flake8 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c6e3ccae346445200894ca4b162511bbbe60c2ff..91c184efe263a3ba1a2498d9d35d9c041383be3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,8 +8,10 @@ # important... for the rascil part to work, you'll have to get rascal-data and also run # git-lfs pull as explained in https://ska-telescope.gitlab.io/rascil/RASCIL_install.html -git+https://gitlab.com/ska-telescope/external/rascil.git@master#egg=rascil - +--index-url https://artefact.skao.int/repository/pypi-internal/simple +--extra-index-url https://pypi.org/simple +ska-sdp-datamodels +git+https://gitlab.com/ska-telescope/external/rascil-main.git@master#egg=rascil pycraf xlrd numpy diff --git a/rfi/common_tools.py b/rfi/common_tools.py index 3bac8c7c3f3a23936a5e4509af59d6e5beb86901..b87c900698592342c8aebe9ad83581771fc2fa43 100644 --- a/rfi/common_tools.py +++ b/rfi/common_tools.py @@ -5,9 +5,9 @@ from typing import Tuple import numpy as np from astropy import units as u from astropy.coordinates import EarthLocation -from rascil.processing_components import ( +from ska_sdp_datamodels.configuration import create_named_configuration +from ska_sdp_datamodels.configuration.config_create import ( create_configuration_from_LLAfile, - create_named_configuration, ) LOG = logging.getLogger("rfi-logger") diff --git a/rfi/pycraf_scripts/SKA_low_pycraf_propagation.py b/rfi/pycraf_scripts/SKA_low_pycraf_propagation.py index 9c23cbbcf9ee41e7a33ea7225b9acf1a4236987a..31eed23626c3f8b2f74b3fbeddec09586ec5be7c 100644 --- a/rfi/pycraf_scripts/SKA_low_pycraf_propagation.py +++ b/rfi/pycraf_scripts/SKA_low_pycraf_propagation.py @@ -19,7 +19,7 @@ from astropy import ( units as u, # must be after pycraf to avoid astropy import conflicts ) from pycraf import antenna, geometry, pathprof -from rascil.processing_components import select_configuration +from ska_sdp_datamodels.configuration import select_configuration from rfi.common_tools import generate_ska_antenna_configuration diff --git a/rfi/rascil_scripts/power_spectrum.py b/rfi/rascil_scripts/power_spectrum.py index 75fe19f6d49f3fa93eb9c7c01dc205c0afd78c4e..50590adda3383c38e9ad5f10bf524180b90b93a4 100644 --- a/rfi/rascil_scripts/power_spectrum.py +++ b/rfi/rascil_scripts/power_spectrum.py @@ -10,12 +10,12 @@ import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt import numpy -from rascil.data_models.xarray_coordinate_support import griddata_wcs -from rascil.processing_components import ( +from rascil.processing_components.image.operations import ( fft_image_to_griddata, import_image_from_fits, show_image, ) +from ska_sdp_datamodels.xarray_coordinate_support import griddata_wcs def radial_profile(image, centre=None): diff --git a/rfi/rascil_scripts/simulate_low_rfi_visibility_propagation.py b/rfi/rascil_scripts/simulate_low_rfi_visibility_propagation.py index c81a2b271536767c5dc389319a6d9bb4ea65c7b0..71ee838d9cd4f417fbd77553905e38233a90d0b2 100644 --- a/rfi/rascil_scripts/simulate_low_rfi_visibility_propagation.py +++ b/rfi/rascil_scripts/simulate_low_rfi_visibility_propagation.py @@ -48,23 +48,23 @@ import matplotlib import numpy from astropy import units as u from astropy.coordinates import SkyCoord -from rascil.data_models.polarisation import PolarisationFrame -from rascil.processing_components.simulation.configurations import ( - select_configuration, -) from rascil.processing_components.simulation.rfi import ( calculate_averaged_correlation, simulate_rfi_block_prop, ) from rascil.processing_components.util.array_functions import average_chunks from rascil.processing_components.visibility.base import ( - create_blockvisibility, - export_blockvisibility_to_ms, + export_visibility_to_ms, ) from rascil.workflows.rsexecute.execution_support.rsexecute import rsexecute -from rascil.workflows.rsexecute.visibility.blockvisibility_rsexecute import ( - concatenate_blockvisibility_time_rsexecute, +from rascil.workflows.rsexecute.visibility.visibility_rsexecute import ( + concatenate_visibility_time_rsexecute, +) +from ska_sdp_datamodels.configuration.config_create import select_configuration +from ska_sdp_datamodels.science_data_model.polarisation_model import ( + PolarisationFrame, ) +from ska_sdp_datamodels.visibility import create_visibility from rfi.common_tools import generate_ska_antenna_configuration from rfi.old_run_checker import check_for_old_run, write_settings @@ -210,8 +210,8 @@ def add_noise(bvis): Add noise to the visibilities. Note: this will update the original bvis input array in-place too. - :param bvis: BlockVisibility - :return: BlockVisibility with noise + :param bvis: Visibility + :return: Visibility with noise """ # The specified sensitivity: # (effective area / T_sys) is roughly 610 m ^ 2 / K in the range 160 - 200MHz # noqa: E501 @@ -244,7 +244,7 @@ def simulate_rfi_image_prop( ): """ Simulate RFI and then average over time and frequency, - producing a BlockVisibility + producing a Visibility :param config: RASCIL telescope Configuration e.g. LOW :param times: observation times (hour angles in radians) @@ -270,7 +270,7 @@ def simulate_rfi_image_prop( :param rfi_frequencies: 1D numpy array of the frequencies where the RFI signal appears - :return: BlockVisibility containing averaged data + :return: Visibility containing averaged data """ averaged_frequency = numpy.array( @@ -287,9 +287,9 @@ def simulate_rfi_image_prop( )[0] # times are the hour angles in seconds (!) relative to transit. - # create_blockvisibility needs hour angles in radians + # create_visibility needs hour angles in radians s2r = numpy.pi / 43200.0 - bvis = create_blockvisibility( + bvis = create_visibility( config, s2r * times, frequency, @@ -299,7 +299,7 @@ def simulate_rfi_image_prop( zerow=False, ) - # Now fill in the blockvisibility with simulated data + # Now fill in the visibility with simulated data bvis = simulate_rfi_block_prop( bvis, apparent_transmitter_power, @@ -310,8 +310,8 @@ def simulate_rfi_image_prop( apply_primary_beam=False, # not needed for Low ) - # Now create a blockvisibility for the averaged data - averaged_bvis = create_blockvisibility( + # Now create a visibility for the averaged data + averaged_bvis = create_visibility( config, s2r * averaged_times, averaged_frequency, @@ -605,11 +605,11 @@ def rfi_simulation(args): ) vis_graph_list.append(vis_graph) - entire_bvis = concatenate_blockvisibility_time_rsexecute(vis_graph_list) + entire_bvis = concatenate_visibility_time_rsexecute(vis_graph_list) entire_bvis = rsexecute.compute(entire_bvis, sync=True) if args.write_ms: - export_blockvisibility_to_ms(args.msout, [entire_bvis]) + export_visibility_to_ms(args.msout, [entire_bvis]) log.info("\nSKA LOW RFI simulation using RASCIL") log.info("Started at {}".format(start_epoch)) diff --git a/rfi/rfi_sim.sh b/rfi/rfi_sim.sh index 12ff4e8ff8fc5206a0d7a8a4ecff0633eebe104d..c8b9ddecd7924eee72d71e4bebaa7b7624d5ccea 100755 --- a/rfi/rfi_sim.sh +++ b/rfi/rfi_sim.sh @@ -80,7 +80,7 @@ echo " Running RASCIL imager... " echo "--------------------------" python3 $RASCIL/rascil/apps/rascil_imager.py --ingest_msname ./simulate_rfi.ms --ingest_dd 0 \ ---ingest_vis_nchan 8 --ingest_chan_per_blockvis 1 --mode invert --imaging_cellsize 0.001 \ +--ingest_vis_nchan 8 --ingest_chan_per_vis 1 --mode invert --imaging_cellsize 0.001 \ --imaging_npixel 2048 || exit echo "--------------------------" diff --git a/rfi/rfi_sim_sim897.sh b/rfi/rfi_sim_sim897.sh index a4c045dd07b51883c66ee12d13ffade0155fee9e..4df00e8f8c87c64d63259dbd61506059f374cb5d 100755 --- a/rfi/rfi_sim_sim897.sh +++ b/rfi/rfi_sim_sim897.sh @@ -81,7 +81,7 @@ echo "--------------------------" python3 $RASCIL/rascil/apps/rascil_imager.py \ --ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms --ingest_dd 0 \ ---ingest_vis_nchan ${n_chan} --ingest_chan_per_blockvis 32 --mode invert --imaging_cellsize 0.0001 \ +--ingest_vis_nchan ${n_chan} --ingest_chan_per_vis 32 --mode invert --imaging_cellsize 0.0001 \ --imaging_npixel 8192 --imaging_weighting uniform || exit echo "--------------------------" diff --git a/rfi/rfi_sim_test.sh b/rfi/rfi_sim_test.sh index b257b72fcb5e500eefbea300f5860620b03e2e0f..774efcdede4878600e7a7812870f10d1d1b0c9a5 100755 --- a/rfi/rfi_sim_test.sh +++ b/rfi/rfi_sim_test.sh @@ -81,7 +81,7 @@ echo " Running RASCIL imager... " echo "--------------------------" python3 $RASCIL/rascil/apps/rascil_imager.py --ingest_msname simulate_rfi.ms --ingest_dd 0 \ ---ingest_vis_nchan 8 --ingest_chan_per_blockvis 1 --mode invert --imaging_cellsize 0.003 \ +--ingest_vis_nchan 8 --ingest_chan_per_vis 1 --mode invert --imaging_cellsize 0.003 \ --imaging_npixel 768 || exit echo "--------------------------" diff --git a/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh b/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh index f72bb7f1aee5b3af494a75a397e39ba2d2a34024..1b584566fec01acaff56d619578f90568e228c78 100755 --- a/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh +++ b/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh @@ -81,7 +81,7 @@ echo "--------------------------" python3 $RASCIL/rascil/apps/rascil_imager.py \ --ingest_msname ./simulate_rfi.ms --ingest_dd 0 \ ---ingest_vis_nchan ${n_chan} --ingest_chan_per_blockvis 32 --mode invert --imaging_cellsize 0.0001 \ +--ingest_vis_nchan ${n_chan} --ingest_chan_per_vis 32 --mode invert --imaging_cellsize 0.0001 \ --imaging_npixel 8192 --imaging_weighting uniform || exit echo "--------------------------" diff --git a/tests/rfi/rascil_scripts/test_regression_rascil_simulate_low_rfi_visibility_propagation.py b/tests/rfi/rascil_scripts/test_regression_rascil_simulate_low_rfi_visibility_propagation.py index d1be8236c2bdc35a7da27da27cba521a2f2d2df8..efef70cb2d9e59170622588f5a471d20e84b2ad4 100644 --- a/tests/rfi/rascil_scripts/test_regression_rascil_simulate_low_rfi_visibility_propagation.py +++ b/tests/rfi/rascil_scripts/test_regression_rascil_simulate_low_rfi_visibility_propagation.py @@ -14,7 +14,9 @@ from rascil.apps.rascil_imager import cli_parser as imager_cli_parser from rascil.apps.rascil_imager import imager from rascil.apps.rascil_vis_ms import cli_parser as vis_cli_parser from rascil.apps.rascil_vis_ms import visualise -from rascil.processing_components import import_image_from_fits, qa_image +from rascil.processing_components.image.operations import ( + import_image_from_fits, +) from rfi.rascil_scripts.power_spectrum import ( cli_parser as powerspectrum_cli_parser, @@ -139,7 +141,7 @@ def test_rfi_simulation(use_dask, msout): "8", "--ingest_dd", "0", - "--ingest_chan_per_blockvis", + "--ingest_chan_per_vis", "1", "--imaging_npixel", "768", @@ -150,7 +152,7 @@ def test_rfi_simulation(use_dask, msout): dirtyname = imager(args) dirty = import_image_from_fits(dirtyname) - qa = qa_image(dirty) + qa = dirty.image_acc.qa_image() assert dirty["pixels"].shape == ( 1, diff --git a/tests/rfi/rascil_scripts/test_unit_simulate_low_rfi_visibility_prop.py b/tests/rfi/rascil_scripts/test_unit_simulate_low_rfi_visibility_prop.py index 03a9c8c70a827fb8d415c38deb98d3358daced5e..6008e1ff56a783692ef652f5520b22d8904a3de4 100644 --- a/tests/rfi/rascil_scripts/test_unit_simulate_low_rfi_visibility_prop.py +++ b/tests/rfi/rascil_scripts/test_unit_simulate_low_rfi_visibility_prop.py @@ -5,11 +5,11 @@ from unittest.mock import Mock, patch import astropy.units as u import numpy as np from astropy.coordinates import SkyCoord -from rascil.data_models import PolarisationFrame -from rascil.processing_components import ( - create_blockvisibility, +from ska_sdp_datamodels.configuration.config_create import ( create_named_configuration, ) +from ska_sdp_datamodels.science_data_model import PolarisationFrame +from ska_sdp_datamodels.visibility import create_visibility from rfi.rascil_scripts.simulate_low_rfi_visibility_propagation import ( add_noise, @@ -41,7 +41,7 @@ def test_add_noise(): ) channel_bandwidth = np.array([1e7, 1e7]) - vis = create_blockvisibility( + vis = create_visibility( config, times, frequency,