Commit e303b591 authored by David Hendriks's avatar David Hendriks
Browse files

updated scaling plots and function to compare the grid runs. i need to keep in... 

updated scaling plots and function to compare the grid runs. i need to keep in mind that these calculations are very probably limited the the physical amount of cores the machine has
parent 194e871b

binarycpython/utils/grid.py

+140 −33
Original line number Diff line number Diff line
@@ -794,40 +794,147 @@ class Population(object): 
            )

        )



        # quit()

        # ########################

    def test_evolve_population_lin(self):

        """

        Test function to evolve a population in a linear way.



        returns total time spent on the actual interfacing with binaryc 

        """



        import time



        #######################

        ### Custom logging code:

        self.set_custom_logging()



        ### Load store

        self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")



        #######################

        # Dry run and getting starcount

        self.grid_options['probtot'] = 0



        self.generate_grid_code(dry_run=True)



        self.load_grid_function()



        self.dry_run()



        total_starcount_run = self.grid_options['total_starcount']

        print("Total starcount for this run will be: {}".format(total_starcount_run))



        #######################

        # Linear run

        start_lin = time.time()



        self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way

 

        self.generate_grid_code(dry_run=False)



        self.load_grid_function()



        for i, system in enumerate(self.grid_options["system_generator"](self)):

            full_system_dict = self.bse_options.copy()

            full_system_dict.update(system)



            binary_cmdline_string = self.return_argline(full_system_dict)

            out = binary_c_python_api.run_population(

                binary_cmdline_string,

                self.grid_options["custom_logging_func_memaddr"],

                self.grid_options["store_memaddr"],

            )

            print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)



        stop_lin = time.time()



        print(

            "Without mp: {} systems took {}s".format(total_starcount_run, stop_lin-start_lin))



        return stop_lin-start_lin





    def test_evolve_population_mp(self):

        """

        Test function to evolve a population in a parallel way.



        returns total time spent on the actual interfacing with binaryc 

        """



        import time

        import multiprocessing as mp

        from pathos.multiprocessing import ProcessingPool as Pool



        #######################

        ### Custom logging code:

        self.set_custom_logging()



        ### Load store

        self.grid_options["store_memaddr"] = binary_c_python_api.return_store("")



        #######################

        # Dry run and getting starcount

        self.grid_options['probtot'] = 0



        # # evolve with mp

        # start_mp = time.time()

        self.generate_grid_code(dry_run=True)



        self.load_grid_function()



        self.dry_run()



        total_starcount_run = self.grid_options['total_starcount']

        print("Total starcount for this run will be: {}".format(total_starcount_run))



        # def evolve_mp(mass):

        #######################

        # MP run

        self.grid_options['probtot'] = 0 # To make sure that the values are reset. TODO: fix this in a cleaner way



        start_mp = time.time()



        self.generate_grid_code(dry_run=False)



        self.load_grid_function()



        def evolve_system(binary_cmdline_string):

            # print(binary_cmdline_string)

            # pass

            # print('next')

            # self.set_bse_option("M_1", mass)

        #     out = binary_c_python_api.run_population(

        #         self.return_argline(),

        #         self.grid_options["custom_logging_func_memaddr"],

        #         self.grid_options["store_memaddr"],

        #     )

        #     parse_function(self, out)

            out = binary_c_python_api.run_population(

                binary_cmdline_string,

                self.grid_options["custom_logging_func_memaddr"],

                self.grid_options["store_memaddr"],

            )

            # # parse_function(self, out)



        # p = Pool(nodes=self.grid_options["amt_cores"])

        def yield_system():

            for i, system in enumerate(self.grid_options["system_generator"](self)):

                full_system_dict = self.bse_options.copy()

                full_system_dict.update(system)



        # def g(mass_distribution):

        #     masses = mass_distribution

        #     for mass in masses:

        #         yield mass

        #     print("generator done")

                binary_cmdline_string = self.return_argline(full_system_dict)

                print("{}/{}".format(i+1, total_starcount_run), binary_cmdline_string)



        # r = list(p.imap(evolve_mp, g(mass_distribution)))

        # stop_mp = time.time()

                yield binary_cmdline_string

            print("generator done")



        # print(

        #     "with mp: {} systems took {}s using {} cores".format(

        #         len(mass_distribution),

        #         stop_mp - start_mp,

        #         self.grid_options["amt_cores"],

        #     )

        # )

        # Create pool

        p = Pool(nodes=self.grid_options["amt_cores"])



        # Execute

        r = list(p.imap(evolve_system, yield_system()))



        stop_mp = time.time()



        # Give feedback

        print(

            "with mp: {} systems took {}s using {} cores".format(

                self.grid_options['total_starcount'],

                stop_mp - start_mp,

                self.grid_options["amt_cores"],

            )

        )



        return stop_mp - start_mp



    def test_evolve_single(self):

        """

tests/population/grid_tests.py

+0 −1
Original line number Diff line number Diff line
@@ -289,7 +289,6 @@ test_pop.set( 


# ###

# testing population:



test_pop.set(verbose=1, 

    amt_cores=1,

    binary=0,

tests/population/scaling/evolve_population_comparing_with_multiprocessing.py

0 → 100644
+143 −0
Original line number Diff line number Diff line 
import os

import json

import time

import pickle

import sys

import numpy as np



from binarycpython.utils.grid import Population

from binarycpython.utils.functions import get_help_all, get_help, create_hdf5





# import argparse





# parser = argparse.ArgumentParser()

# parser.add_argument(

#     "amt_systems", help="the amount of systems",

# )



# parser.add_argument(

#     "amt_nodes", help="the amount of nodes that are used for the multiprocessing",

# )



# parser.add_argument(

#     "name_testcase", help="The name of the testcase (e.g. laptop, cluster etc)",

# )





# args = parser.parse_args()



# amt_systems = args.amt_systems

# amt_nodes = args.amt_nodes

# name_testcase = args.name_testcase





## Quick script to get some output about which stars go supernova when.

def output_lines(output):

    """

    Function that outputs the lines that were recieved from the binary_c run. 

    """

    return output.splitlines()





def parse_function(self, output):

    # extract info from the population instance

    # TODO: think about whether this is smart. Passing around this object might be an overkill



    # Get some information from the

    data_dir = self.custom_options["data_dir"]

    base_filename = self.custom_options["base_filename"]



    # Check directory, make if necessary

    os.makedirs(data_dir, exist_ok=True)



    # Create filename

    outfilename = os.path.join(data_dir, base_filename)



    # Go over the output.

    for el in output_lines(output):

        headerline = el.split()[0]



        # CHeck the header and act accordingly

        if headerline == "DAVID_SN":

            parameters = ["time", "mass_1", "prev_mass_1", "zams_mass_1", "SN_type"]

            values = el.split()[1:]

            seperator = "\t"



            if not os.path.exists(outfilename):

                with open(outfilename, "w") as f:

                    f.write(seperator.join(parameters) + "\n")



            with open(outfilename, "a") as f:

                f.write(seperator.join(values) + "\n")



test_pop = Population()



test_pop.set(verbose=1, 

    amt_cores=4,

    binary=1,

)





resolution = {'M_1': 10, 'per': 10}



test_pop.add_grid_variable(

    name="M_1",

    longname="log primary mass",

    valuerange=[10, 100],

    resolution="{}".format(resolution['M_1']),

    spacingfunc="const(10, 100, {})".format(resolution['M_1']),

    precode="",

    # precode="M_1=math.exp(lnm1)",

    probdist="flat(M_1)",

    # probdist='self.custom_options["extra_prob_function"](M_1)',

    dphasevol="dlnm1",

    parameter_name="M_1",

    condition="",

)



### Grid generating test.

test_pop.add_grid_variable(

    name="period",

    longname="period",

    valuerange=["M_1", 20],

    resolution="{}".format(resolution['per']),

    spacingfunc="np.linspace(1, 10, {})".format(resolution['per']),

    precode="orbital_period = period**2",

    probdist="flat(orbital_period)",

    parameter_name="orbital_period",

    dphasevol="dper",

    condition='self.grid_options["binary"]==1',

)



test_pop.set(verbose=1, 

    amt_cores=2,

    binary=1,

)



total_systems = np.prod([el for el in resolution.values()])

AMT_CORES = 2



# Lin

total_lin_start = time.time()



evolve_lin_time = test_pop.test_evolve_population_lin()



total_lin_stop = time.time()



total_lin = total_lin_stop - total_lin_start



print("linear run with {} systems: {} of which {} spent on evolving the systems".format(total_systems, total_lin, evolve_lin_time))



# MP

total_mp_start = time.time()



evolve_mp_time = test_pop.test_evolve_population_mp()



total_mp_stop = time.time()



total_mp = total_mp_stop - total_mp_start

print("MP ({} nodes) run with {} systems: {} of which {} spent on evolving the systems".format(AMT_CORES, total_systems, total_mp, evolve_mp_time))



print("The speed up by using MP is: {}".format(total_lin/total_mp))
 
 No newline at end of file

tests/population/scaling/plot_scaling.py

+3 −3
Original line number Diff line number Diff line
@@ -179,9 +179,9 @@ def plot_runtime(calculated_results, unique_amt_cores, unique_amt_systems): 
##################### 



# Configure

result_file = "comparison_result.dat"

result_file = "comparison_result_astro1.dat"

name_testcase = "Astro1"

result_file = "comparison_result_laptop.dat"

# result_file = "comparison_result_astro1.dat"

name_testcase = "laptop"

img_dir = "scaling_plots"



# Readout file