From 55a2c5d346e812f787de401db245792b29517e04 Mon Sep 17 00:00:00 2001 From: Tim Cornwell <realtimcornwell@gmail.com> Date: Wed, 7 Jul 2021 14:33:38 +0100 Subject: [PATCH 1/6] SIM-930: Update scripts for new scale --- continuum_imaging/sp_1331/image_low.sh | 12 +++ .../sp_1331/image_low_csd3.slurm | 85 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 continuum_imaging/sp_1331/image_low.sh create mode 100644 continuum_imaging/sp_1331/image_low_csd3.slurm diff --git a/continuum_imaging/sp_1331/image_low.sh b/continuum_imaging/sp_1331/image_low.sh new file mode 100644 index 0000000..e79ceaa --- /dev/null +++ b/continuum_imaging/sp_1331/image_low.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# +# Image simulated data +# +results_dir=${HOME}/data/ska-sim-low/continuum_imaging/sp_1331/ +python3 ${RASCIL}/rascil/apps/rascil_imager.py --clean_nmoment 3 --clean_facets 2 --clean_overlap 32 \ + --clean_nmajor 10 --clean_threshold 3e-5 --clean_restore_facets 2 --clean_restore_overlap 32 \ + --use_dask True --imaging_context ng --imaging_npixel 16384 --imaging_pol stokesI --clean_restored_output list \ + --imaging_cellsize 1e-5 --imaging_weighting uniform --imaging_nchan 1 \ + --ingest_vis_nchan 100 --ingest_chan_per_blockvis 6 \ + --ingest_msname ${results_dir}/scalar_gaussian_beams.ms \ + --performance_file ${results_dir}/performance_rascil_imager.json diff --git a/continuum_imaging/sp_1331/image_low_csd3.slurm b/continuum_imaging/sp_1331/image_low_csd3.slurm new file mode 100644 index 0000000..0564cf2 --- /dev/null +++ b/continuum_imaging/sp_1331/image_low_csd3.slurm @@ -0,0 +1,85 @@ +#!/bin/bash +#! +#! Dask job script for CSD3, runs image_B2_5km.sh +#! Tim Cornwell +#! +#!############################################################# +#!#### Modify the options in this section as appropriate ###### +#!############################################################# + +#! sbatch directives begin here ############################### +#! Name of the job: +#SBATCH -J SP1331_LOW +#! Which project should be charged: +#SBATCH -A SKA-SDP-SL2-CPU +#! How many whole nodes should be allocated? +#SBATCH --nodes=16 +#! How many (MPI) tasks will there be in total? (<= nodes*16) +#SBATCH --ntasks=16 +#! Memory limit +##SBATCH --mem 500000 +#! How much wallclock time will be required? +#SBATCH --time=20:00:00 +#! What types of email messages do you wish to receive? +#SBATCH --mail-type=BEGIN,FAIL,END +#! Where to send email messages +#SBATCH --mail-user=realtimcornwell@gmail.com +#! Uncomment this to prevent the job from being requeued (e.g. if +#! interrupted by node failure or system downtime): +##SBATCH --no-requeue +#! Do not change: +#! For 6GB per CPU, set "-p skylake"; for 12GB per CPU, set "-p skylake-himem": +#SBATCH -p skylake-himem +#SBATCH --exclusive +#! Same switch +#SBATCH --switches=1 +#! Uncomment this to prevent the job from being requeued (e.g. if +#! interrupted by node failure or system downtime): +##SBATCH --no-requeue +#! Optionally modify the environment seen by the application +#! (note that SLURM reproduces the environment at submission irrespective of ~/.bashrc): +module purge # Removes all modules still loaded +module load rhel7/default-peta4 # REQUIRED - loads the basic environment +module load gcc +module load slurm + +export SMSROOT=/home/hpccorn1/Code/ska-sim-mid +export RASCIL=/home/hpccorn1/Code/rascil +export RASCIL=$HOME/Code/rascil +export PYTHONPATH=$RASCIL:$PYTHONPATH:$SMSROOT +echo "PYTHONPATH: $PYTHONPATH" + +echo -e "Running python: `which python`" +echo -e "Running dask-scheduler: `which dask-scheduler`" + +module load slurm + +cd $SLURM_SUBMIT_DIR +echo -e "Changed directory to `pwd`.\n" + +JOBID=${SLURM_JOB_ID} +echo ${SLURM_JOB_NODELIST} + +#! Create a hostfile: +scontrol show hostnames $SLURM_JOB_NODELIST | uniq > hostfile.$JOBID +scheduler=$(head -1 hostfile.$JOBID) + +echo "run dask-scheduler" +ssh $scheduler dask-scheduler --port=8786 & +sleep 5 + +for host in `cat hostfile.$JOBID`; do + echo "Working on $host ...." + echo "run dask-worker" + ssh $host dask-worker --host ${host} --nprocs 1 --nthreads 4 \ + --local-directory /tmp \ + --memory-limit 0 $scheduler:8786 & + sleep 1 +done +echo "Scheduler and workers now running" +echo "Scheduler is running at ${scheduler}:8786" + +CMD="RASCIL_DASK_SCHEDULER=${scheduler}:8786 sh image_low.sh" +echo "About to execute $CMD" + +eval $CMD -- GitLab From 90117a8e8931a290d5ff403aeb17fb993f6b99a1 Mon Sep 17 00:00:00 2001 From: Tim Cornwell <realtimcornwell@gmail.com> Date: Sat, 10 Jul 2021 18:48:36 +0100 Subject: [PATCH 2/6] SIM-930: Don't use facets for restore --- continuum_imaging/sp_1331/image_low.sh | 5 +++-- continuum_imaging/sp_1331/image_low_csd3.slurm | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/continuum_imaging/sp_1331/image_low.sh b/continuum_imaging/sp_1331/image_low.sh index e79ceaa..636c39c 100644 --- a/continuum_imaging/sp_1331/image_low.sh +++ b/continuum_imaging/sp_1331/image_low.sh @@ -4,8 +4,9 @@ # results_dir=${HOME}/data/ska-sim-low/continuum_imaging/sp_1331/ python3 ${RASCIL}/rascil/apps/rascil_imager.py --clean_nmoment 3 --clean_facets 2 --clean_overlap 32 \ - --clean_nmajor 10 --clean_threshold 3e-5 --clean_restore_facets 2 --clean_restore_overlap 32 \ - --use_dask True --imaging_context ng --imaging_npixel 16384 --imaging_pol stokesI --clean_restored_output list \ + --clean_nmajor 10 --clean_threshold 3e-5 --clean_restore_facets 1 \ + --use_dask True --imaging_context ng --imaging_npixel 16384 --imaging_pol stokesI \ + --clean_restored_output taylor \ --imaging_cellsize 1e-5 --imaging_weighting uniform --imaging_nchan 1 \ --ingest_vis_nchan 100 --ingest_chan_per_blockvis 6 \ --ingest_msname ${results_dir}/scalar_gaussian_beams.ms \ diff --git a/continuum_imaging/sp_1331/image_low_csd3.slurm b/continuum_imaging/sp_1331/image_low_csd3.slurm index 0564cf2..7f89375 100644 --- a/continuum_imaging/sp_1331/image_low_csd3.slurm +++ b/continuum_imaging/sp_1331/image_low_csd3.slurm @@ -9,7 +9,7 @@ #! sbatch directives begin here ############################### #! Name of the job: -#SBATCH -J SP1331_LOW +#SBATCH -J LOW #! Which project should be charged: #SBATCH -A SKA-SDP-SL2-CPU #! How many whole nodes should be allocated? @@ -19,7 +19,7 @@ #! Memory limit ##SBATCH --mem 500000 #! How much wallclock time will be required? -#SBATCH --time=20:00:00 +#SBATCH --time=23:30:00 #! What types of email messages do you wish to receive? #SBATCH --mail-type=BEGIN,FAIL,END #! Where to send email messages @@ -71,8 +71,7 @@ sleep 5 for host in `cat hostfile.$JOBID`; do echo "Working on $host ...." echo "run dask-worker" - ssh $host dask-worker --host ${host} --nprocs 1 --nthreads 4 \ - --local-directory /tmp \ + ssh $host dask-worker --host ${host} --nprocs 1 --nthreads 1 \ --memory-limit 0 $scheduler:8786 & sleep 1 done -- GitLab From b98222fb9fa4b37a913fabc425917ce95ab56408 Mon Sep 17 00:00:00 2001 From: Tim Cornwell <realtimcornwell@gmail.com> Date: Wed, 14 Jul 2021 12:46:22 +0100 Subject: [PATCH 3/6] SIM-930: Increase number of clean moments --- rfi/rascil_scripts/plot_AA0p5.py | 63 ++++++++++++++++++ rfi/rfi_sim_p3_sim897.slurm | 81 +++++++++++++++++++++++ rfi/rfi_sim_sim897.sh | 96 ++++++++++++++++++++++++++++ rfi/rfi_sims/README.md | 15 +++++ rfi/rfi_sims/aa0p5/README.md | 14 ++++ rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh | 96 ++++++++++++++++++++++++++++ rfi/to_gcp.sh | 9 +++ 7 files changed, 374 insertions(+) create mode 100644 rfi/rascil_scripts/plot_AA0p5.py create mode 100644 rfi/rfi_sim_p3_sim897.slurm create mode 100755 rfi/rfi_sim_sim897.sh create mode 100644 rfi/rfi_sims/README.md create mode 100644 rfi/rfi_sims/aa0p5/README.md create mode 100755 rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh create mode 100644 rfi/to_gcp.sh diff --git a/rfi/rascil_scripts/plot_AA0p5.py b/rfi/rascil_scripts/plot_AA0p5.py new file mode 100644 index 0000000..9b8935c --- /dev/null +++ b/rfi/rascil_scripts/plot_AA0p5.py @@ -0,0 +1,63 @@ + +import matplotlib.pyplot as plt + +import numpy + +from rascil.processing_components import create_named_configuration, plot_configuration, select_configuration + +aa0p5 = create_named_configuration("LOW-AA0.5") + +# all_names = [ +# "S8â€1", +# "S8â€2", +# "S8â€3", +# "S8â€4", +# "S8â€5", +# "S8â€6", +# "S9â€1", +# "S9â€2", +# "S9â€3", +# "S9â€4", +# "S9â€5", +# "S9â€6", +# "S10â€1", +# "S10â€2", +# "S10â€3", +# "S10â€4", +# "S10â€5", +# "S10â€6", +# ] +# +# for cluster in range(3): +# aasel=aa0p5.sel({"id": range(cluster*6, (cluster+1)*6)}) +# plot_configuration(aasel, label=True, title=f"aa0p5_S{cluster+8}") +# plt.show(block=False) +# plt.savefig(f"aa0p5_S{cluster+8}.jpg") +# +# sel_names = [ +# # "S8â€1", +# # "S8â€2", +# # "S8â€3", +# "S8â€4", +# "S8â€5", +# # "S8â€6", +# # "S9â€1", +# # "S9â€2", +# "S9â€3", +# # "S9â€4", +# # "S9â€5", +# "S9â€6", +# "S10â€1", +# # "S10â€2", +# # "S10â€3", +# # "S10â€4", +# "S10â€5", +# # "S10â€6", +# ] +# +# #aasel = aa0p5.sel({"names":sel_names}) +# # aasel = aa0p5.sel({"id":[3, 4, 8, 11, 12, 16]}) +# # print(aasel["names"]) +plot_configuration(aa0p5, label=True, title=f"aa0p5") +plt.show(block=False) +plt.savefig(f"aa0p5.jpg") diff --git a/rfi/rfi_sim_p3_sim897.slurm b/rfi/rfi_sim_p3_sim897.slurm new file mode 100644 index 0000000..4cef4be --- /dev/null +++ b/rfi/rfi_sim_p3_sim897.slurm @@ -0,0 +1,81 @@ +#!/bin/bash +#! +#! Dask job script for P3 +#! Tim Cornwell +#! + +#!############################################################# +#!#### Modify the options in this section as appropriate ###### +#!############################################################# + +#! sbatch directives begin here ############################### +#! Name of the job: +#SBATCH -J SDP_RASCIL +#! Which project should be charged: +#SBATCH -A SKA-SDP +#! How many whole nodes should be allocated? +#SBATCH --nodes=8 +#! How many (MPI) tasks will there be in total? (<= nodes*16) +#SBATCH --ntasks=64 +#! Memory limit +#SBATCH --mem 128000 +#! How much wallclock time will be required? +#SBATCH --time=10:00:00 +#! What types of email messages do you wish to receive? +#SBATCH --mail-type=FAIL +#! Uncomment this to prevent the job from being requeued (e.g. if +#! interrupted by node failure or system downtime): +##SBATCH --no-requeue +#! Do not change: +#SBATCH -p compute +#! What types of email messages do you wish to receive? +#SBATCH --mail-type=FAIL +#! Uncomment this to prevent the job from being requeued (e.g. if +#! interrupted by node failure or system downtime): +##SBATCH --no-requeue +# avoid dask schedulers to be confused by requiring exclusive nodes +#SBATCH --exclusive +#! Modify the settings below to specify the application's environment, location +#! and launch method: + +#! Optionally modify the environment seen by the application +#! (note that SLURM reproduces the environment at submission irrespective of ~/.bashrc): +#module purge # Removes all modules still loaded + +#! Set up python +export PYTHONPATH=$PYTHONPATH:$RASCIL +echo "PYTHONPATH is ${PYTHONPATH}" + +echo -e "Running python: `which python`" +echo -e "Running dask-scheduler: `which dask-scheduler`" + +cd $SLURM_SUBMIT_DIR +echo -e "Changed directory to `pwd`.\n" + +JOBID=${SLURM_JOB_ID} +echo ${SLURM_JOB_NODELIST} + +#! Create a hostfile: +scontrol show hostnames $SLURM_JOB_NODELIST | uniq > hostfile.$JOBID + +scheduler=$(head -1 hostfile.$JOBID) +hostIndex=0 +for host in `cat hostfile.$JOBID`; do + echo "Working on $host ...." + if [ "$hostIndex" = "0" ]; then + echo "run dask-scheduler" + ssh $host dask-scheduler --port=8786 & + sleep 5 + fi + echo "run dask-worker" + ssh $host dask-worker --host ${host} --nprocs 8 --nthreads 4 \ + --memory-limit 0 $scheduler:8786 & + sleep 1 + hostIndex="1" +done +echo "Scheduler and workers now running" + +CMD="RASCIL_DASK_SCHEDULER=${scheduler}:8786 sh $1 | tee rfi_sim_slurm.log" +echo "About to execute $CMD" + +eval $CMD diff --git a/rfi/rfi_sim_sim897.sh b/rfi/rfi_sim_sim897.sh new file mode 100755 index 0000000..a4c045d --- /dev/null +++ b/rfi/rfi_sim_sim897.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Common settings for running an end-to-end simulation. +# See the python3 help for more details about these options. + +# necessary for imports to work +# we are adding the git directory ska-sim-low to PYTHONPATH +git_root_dir=`git rev-parse --show-toplevel` +export PYTHONPATH="${git_root_dir}:$PYTHONPATH" + +# RA +ra=0.0 +# Dec +dec=-90 + +n_chan="512" +# Number of channels to average +ch_avg="1" +# Pre-averaging integration time +t_int="1.0" +# Number of integrations per chunk +n_t=16 +# Number of integrations to average +t_avg="1" +# Hour angle range (in hours) +t_range_beg=-4.0 +t_range_end=4.0 + +# Number of transmitters +n_trans="100" +freq_range_mhz_low="170.5" #total range -- lower end +freq_range_mhz_high="184.5" #total range -- upper end + +atten_dir="./data/attenuation/" +beamgain_dir="./data/beam_gain/" +pycraf_file="./data/transmitters/Filtered_DTV_list_1.csv" +trans_out_file="./data/attenuation/DTV_above0_1775_${n_trans}T_${n_chan}C.csv" +antenna_workfile="./data/telescope_files/SKA1-LOW_SKO-0000422_Rev3_38m_SKALA4_spot_frequencies.tm/layout_wgs84.txt" +telescope_path='data/telescope_files/SKA1-LOW_SKO-0000422_Rev3_38m_SKALA4_spot_frequencies.tm' + +# Run + +echo "--------------------------" +echo " Running pycraf script... " +echo "--------------------------" + +python ./pycraf_scripts/SKA_low_RFI_propagation.py --transmitters ${pycraf_file} --output_dir ${atten_dir} \ + --n_channels ${n_chan} --frequency_range ${freq_range_mhz_low} ${freq_range_mhz_high} --trans_out ${trans_out_file} \ + --srtm_directory ./data/SRTM_data/ --antenna_file ${antenna_workfile} || exit + +echo "--------------------------" +echo " Running OSKAR script... " +echo "--------------------------" + +python3 ./oskar_sim_beam_gain_sing.py --transmitters ${trans_out_file} --outdir ${beamgain_dir} --indir ${atten_dir} \ + --oskar_path ./OSKAR-2.7.6-Python3.sif --N_channels ${n_chan} \ + --frequency_range ${freq_range_mhz_low} ${freq_range_mhz_high} \ + --ra ${ra} --dec ${dec} --telescope_path ${telescope_path}|| exit + +echo "--------------------------" +echo " Running RASCIL script... " +echo "--------------------------" + +python3 ./rascil_scripts/simulate_low_rfi_visibility_propagation.py --use_dask=True \ + --telescope LOW-AA0.5 --rmax 1e4 --noise True --use_agg True --ra ${ra} --declination ${dec} --use_beamgain True \ + --beamgain_dir ${beamgain_dir} --attenuation_dir ${atten_dir} \ + --transmitter_file ${trans_out_file} --time_range ${t_range_beg} ${t_range_end} --channel_average ${ch_avg} \ + --nintegrations_per_chunk ${n_t} \ + --nchannels_per_chunk ${n_chan} --integration_time ${t_int} --time_average ${t_avg} --write_ms True || exit + +sleep 30 + +echo "--------------------------" +echo " Finding MS size... " +echo "--------------------------" +echo `du -sh /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms` +echo +echo "--------------------------" +echo " Running RASCIL imager... " +echo "--------------------------" + +python3 $RASCIL/rascil/apps/rascil_imager.py \ +--ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms --ingest_dd 0 \ +--ingest_vis_nchan ${n_chan} --ingest_chan_per_blockvis 32 --mode invert --imaging_cellsize 0.0001 \ +--imaging_npixel 8192 --imaging_weighting uniform || exit + +echo "--------------------------" +echo " Running RASCIL visualise " +echo "--------------------------" + +python3 $RASCIL/rascil/apps/rascil_vis_ms.py \ + --ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms || exit + +echo "--------------------------" +echo " Finished. " +echo "--------------------------" diff --git a/rfi/rfi_sims/README.md b/rfi/rfi_sims/README.md new file mode 100644 index 0000000..0807b1f --- /dev/null +++ b/rfi/rfi_sims/README.md @@ -0,0 +1,15 @@ + +The SKA direction-dependent RFI simulations are kept on the Google Cloud Platform. + +The python command line tool gsutil allows for interacting with the Google Cloud Platform: + + https://cloud.google.com/storage/docs/gsutil + +After installing gsutil, you may download the simulations as follows: + + mkdir rfi_sims + cd rfi_sims + gsutil rsync -r gs://ska1-simulation-data/ska1-low/rfi_sims . + +If you wish to run these or similar simulations, install RASCIL, and use the +shell scripts and slurm files as template. diff --git a/rfi/rfi_sims/aa0p5/README.md b/rfi/rfi_sims/aa0p5/README.md new file mode 100644 index 0000000..e813a20 --- /dev/null +++ b/rfi/rfi_sims/aa0p5/README.md @@ -0,0 +1,14 @@ + +The SKA direction-dependent RFI simulations are kept on the Google Cloud Platform. + +The python command line tool gsutil allows for interacting with the Google Cloud Platform: + + https://cloud.google.com/storage/docs/gsutil + +After installing gsutil, you may download the simulations as follows: + + cd rfi_sims + gsutil rsync gs://ska1-simulation-data/ska1-low/rfi_sims . + +If you wish to run these or similar simulations, install RASCIL, and use the +shell scripts and slurm files as template. diff --git a/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh b/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh new file mode 100755 index 0000000..a4c045d --- /dev/null +++ b/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# Common settings for running an end-to-end simulation. +# See the python3 help for more details about these options. + +# necessary for imports to work +# we are adding the git directory ska-sim-low to PYTHONPATH +git_root_dir=`git rev-parse --show-toplevel` +export PYTHONPATH="${git_root_dir}:$PYTHONPATH" + +# RA +ra=0.0 +# Dec +dec=-90 + +n_chan="512" +# Number of channels to average +ch_avg="1" +# Pre-averaging integration time +t_int="1.0" +# Number of integrations per chunk +n_t=16 +# Number of integrations to average +t_avg="1" +# Hour angle range (in hours) +t_range_beg=-4.0 +t_range_end=4.0 + +# Number of transmitters +n_trans="100" +freq_range_mhz_low="170.5" #total range -- lower end +freq_range_mhz_high="184.5" #total range -- upper end + +atten_dir="./data/attenuation/" +beamgain_dir="./data/beam_gain/" +pycraf_file="./data/transmitters/Filtered_DTV_list_1.csv" +trans_out_file="./data/attenuation/DTV_above0_1775_${n_trans}T_${n_chan}C.csv" +antenna_workfile="./data/telescope_files/SKA1-LOW_SKO-0000422_Rev3_38m_SKALA4_spot_frequencies.tm/layout_wgs84.txt" +telescope_path='data/telescope_files/SKA1-LOW_SKO-0000422_Rev3_38m_SKALA4_spot_frequencies.tm' + +# Run + +echo "--------------------------" +echo " Running pycraf script... " +echo "--------------------------" + +python ./pycraf_scripts/SKA_low_RFI_propagation.py --transmitters ${pycraf_file} --output_dir ${atten_dir} \ + --n_channels ${n_chan} --frequency_range ${freq_range_mhz_low} ${freq_range_mhz_high} --trans_out ${trans_out_file} \ + --srtm_directory ./data/SRTM_data/ --antenna_file ${antenna_workfile} || exit + +echo "--------------------------" +echo " Running OSKAR script... " +echo "--------------------------" + +python3 ./oskar_sim_beam_gain_sing.py --transmitters ${trans_out_file} --outdir ${beamgain_dir} --indir ${atten_dir} \ + --oskar_path ./OSKAR-2.7.6-Python3.sif --N_channels ${n_chan} \ + --frequency_range ${freq_range_mhz_low} ${freq_range_mhz_high} \ + --ra ${ra} --dec ${dec} --telescope_path ${telescope_path}|| exit + +echo "--------------------------" +echo " Running RASCIL script... " +echo "--------------------------" + +python3 ./rascil_scripts/simulate_low_rfi_visibility_propagation.py --use_dask=True \ + --telescope LOW-AA0.5 --rmax 1e4 --noise True --use_agg True --ra ${ra} --declination ${dec} --use_beamgain True \ + --beamgain_dir ${beamgain_dir} --attenuation_dir ${atten_dir} \ + --transmitter_file ${trans_out_file} --time_range ${t_range_beg} ${t_range_end} --channel_average ${ch_avg} \ + --nintegrations_per_chunk ${n_t} \ + --nchannels_per_chunk ${n_chan} --integration_time ${t_int} --time_average ${t_avg} --write_ms True || exit + +sleep 30 + +echo "--------------------------" +echo " Finding MS size... " +echo "--------------------------" +echo `du -sh /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms` +echo +echo "--------------------------" +echo " Running RASCIL imager... " +echo "--------------------------" + +python3 $RASCIL/rascil/apps/rascil_imager.py \ +--ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms --ingest_dd 0 \ +--ingest_vis_nchan ${n_chan} --ingest_chan_per_blockvis 32 --mode invert --imaging_cellsize 0.0001 \ +--imaging_npixel 8192 --imaging_weighting uniform || exit + +echo "--------------------------" +echo " Running RASCIL visualise " +echo "--------------------------" + +python3 $RASCIL/rascil/apps/rascil_vis_ms.py \ + --ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms || exit + +echo "--------------------------" +echo " Finished. " +echo "--------------------------" diff --git a/rfi/to_gcp.sh b/rfi/to_gcp.sh new file mode 100644 index 0000000..963b721 --- /dev/null +++ b/rfi/to_gcp.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Upload images to GCP + +set -x + +gsutil rsync -r rfi_sims gs://ska1-simulation-data/ska1-low/rfi_sims + +# The rfi sims should be at: ska1-simulation-data/ska1-low/rfi_sims +gsutil -m acl -r ch -g AllUsers:R gs://ska1-simulation-data/ska1-low/rfi_sims/aa0p5/*.ms -- GitLab From f7db7e7f70822e7d6bdd31e98079ccbca34aa05d Mon Sep 17 00:00:00 2001 From: Tim Cornwell <realtimcornwell@gmail.com> Date: Wed, 14 Jul 2021 13:04:45 +0100 Subject: [PATCH 4/6] SIM-930: Blacker --- rfi/rascil_scripts/plot_AA0p5.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rfi/rascil_scripts/plot_AA0p5.py b/rfi/rascil_scripts/plot_AA0p5.py index 9b8935c..0169d0b 100644 --- a/rfi/rascil_scripts/plot_AA0p5.py +++ b/rfi/rascil_scripts/plot_AA0p5.py @@ -1,9 +1,12 @@ - import matplotlib.pyplot as plt import numpy -from rascil.processing_components import create_named_configuration, plot_configuration, select_configuration +from rascil.processing_components import ( + create_named_configuration, + plot_configuration, + select_configuration, +) aa0p5 = create_named_configuration("LOW-AA0.5") -- GitLab From e3f98d77fc20cc2cedbdccae08ed4d0dba5c1c62 Mon Sep 17 00:00:00 2001 From: Tim Cornwell <realtimcornwell@gmail.com> Date: Wed, 14 Jul 2021 16:43:33 +0100 Subject: [PATCH 5/6] SIM-930: Remove AA0p5 plot file --- .gitignore | 4 ++ rfi/rascil_scripts/plot_AA0p5.py | 66 -------------------------------- 2 files changed, 4 insertions(+), 66 deletions(-) delete mode 100644 rfi/rascil_scripts/plot_AA0p5.py diff --git a/.gitignore b/.gitignore index f11f24a..d914f19 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,7 @@ docs/apidocs .vscode rfi/dask-worker-space/ + +rfi/data/attenuation/tv_transmitter_attenuation_cube.hdf5 + +rfi/rascil_scripts/old_settings/ diff --git a/rfi/rascil_scripts/plot_AA0p5.py b/rfi/rascil_scripts/plot_AA0p5.py deleted file mode 100644 index 0169d0b..0000000 --- a/rfi/rascil_scripts/plot_AA0p5.py +++ /dev/null @@ -1,66 +0,0 @@ -import matplotlib.pyplot as plt - -import numpy - -from rascil.processing_components import ( - create_named_configuration, - plot_configuration, - select_configuration, -) - -aa0p5 = create_named_configuration("LOW-AA0.5") - -# all_names = [ -# "S8â€1", -# "S8â€2", -# "S8â€3", -# "S8â€4", -# "S8â€5", -# "S8â€6", -# "S9â€1", -# "S9â€2", -# "S9â€3", -# "S9â€4", -# "S9â€5", -# "S9â€6", -# "S10â€1", -# "S10â€2", -# "S10â€3", -# "S10â€4", -# "S10â€5", -# "S10â€6", -# ] -# -# for cluster in range(3): -# aasel=aa0p5.sel({"id": range(cluster*6, (cluster+1)*6)}) -# plot_configuration(aasel, label=True, title=f"aa0p5_S{cluster+8}") -# plt.show(block=False) -# plt.savefig(f"aa0p5_S{cluster+8}.jpg") -# -# sel_names = [ -# # "S8â€1", -# # "S8â€2", -# # "S8â€3", -# "S8â€4", -# "S8â€5", -# # "S8â€6", -# # "S9â€1", -# # "S9â€2", -# "S9â€3", -# # "S9â€4", -# # "S9â€5", -# "S9â€6", -# "S10â€1", -# # "S10â€2", -# # "S10â€3", -# # "S10â€4", -# "S10â€5", -# # "S10â€6", -# ] -# -# #aasel = aa0p5.sel({"names":sel_names}) -# # aasel = aa0p5.sel({"id":[3, 4, 8, 11, 12, 16]}) -# # print(aasel["names"]) -plot_configuration(aa0p5, label=True, title=f"aa0p5") -plt.show(block=False) -plt.savefig(f"aa0p5.jpg") -- GitLab From 5ac02fbe3248e92dd7c966fe2f7ffad70f42ebf9 Mon Sep 17 00:00:00 2001 From: Tim Cornwell <realtimcornwell@gmail.com> Date: Thu, 15 Jul 2021 10:46:04 +0100 Subject: [PATCH 6/6] SIM-930: Remove /alaska/tim.... from rfi script --- rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh b/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh index a4c045d..f72bb7f 100755 --- a/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh +++ b/rfi/rfi_sims/aa0p5/rfi_sim_sim897.sh @@ -73,14 +73,14 @@ sleep 30 echo "--------------------------" echo " Finding MS size... " echo "--------------------------" -echo `du -sh /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms` +echo `du -sh ./simulate_rfi.ms` echo echo "--------------------------" echo " Running RASCIL imager... " echo "--------------------------" python3 $RASCIL/rascil/apps/rascil_imager.py \ ---ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms --ingest_dd 0 \ +--ingest_msname ./simulate_rfi.ms --ingest_dd 0 \ --ingest_vis_nchan ${n_chan} --ingest_chan_per_blockvis 32 --mode invert --imaging_cellsize 0.0001 \ --imaging_npixel 8192 --imaging_weighting uniform || exit @@ -89,7 +89,7 @@ echo " Running RASCIL visualise " echo "--------------------------" python3 $RASCIL/rascil/apps/rascil_vis_ms.py \ - --ingest_msname /alaska/tim/Code/ska-sim-low/rfi/simulate_rfi.ms || exit + --ingest_msname ./simulate_rfi.ms || exit echo "--------------------------" echo " Finished. " -- GitLab